You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/05 00:09:42 UTC
[01/21] kylin git commit: KYLIN-976 very initial
Repository: kylin
Updated Branches:
refs/heads/2.x-staging b8ee69c7e -> 7561d80c9
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index fc1d967..7a75bbc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index b7b12d1..b75510f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
index 634e7d3..c9343fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
@@ -25,13 +25,13 @@ import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
index dc33b27..a9d3b10 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
@@ -21,7 +21,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.util.Map;
import java.util.Set;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 95cca0d..8f0b17b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -22,15 +22,15 @@ import java.nio.ByteBuffer;
import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.invertedindex.index.RawTableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index e4360a2..046d8c1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.CompressionUtils;
@@ -47,7 +48,6 @@ import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
@@ -65,6 +65,7 @@ import com.google.protobuf.HBaseZeroCopyByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
index 509df42..c834ee2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -33,7 +34,6 @@ import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 8cb8f13..691258d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
@@ -58,7 +59,6 @@ import org.apache.kylin.engine.mr.IMROutput2;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.steps.MergeCuboidMapper;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
index 7f40259..4037f90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
@@ -5,11 +5,11 @@ import java.util.List;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index 50c2fac..54217ff 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -23,11 +23,11 @@ import java.util.BitSet;
import java.util.Collection;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index a5aba2c..08d4a49 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
index 8b56605..8e665f7 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
@@ -19,14 +19,15 @@
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import com.google.common.collect.Lists;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index 270928f..cf706e4 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -29,13 +29,13 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index 68a2e76..d534e44 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -25,14 +25,14 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
[21/21] kylin git commit: KYLIN-976 fix compile error after rebase
Posted by li...@apache.org.
KYLIN-976 fix compile error after rebase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7561d80c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7561d80c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7561d80c
Branch: refs/heads/2.x-staging
Commit: 7561d80c966a68b3630940a55f0421e43c4c59b2
Parents: b0643f9
Author: Yang Li <li...@apache.org>
Authored: Sat Dec 5 07:08:46 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 07:08:46 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/model/v2/CubeDesc.java | 13 ++++++-------
.../kylin/cube/model/validation/rule/FunctionRule.java | 4 ----
.../kylin/job/upgrade/v2/DataModelDescUpgradeV2.java | 2 +-
.../storage/hbase/cube/v2/CubeSegmentScanner.java | 2 +-
4 files changed, 8 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7561d80c/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
index b37e5a1..30b977d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v2/CubeDesc.java
@@ -41,6 +41,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -648,10 +649,7 @@ public class CubeDesc extends RootPersistentEntity {
func.init(factTable);
allColumns.addAll(func.getParameter().getColRefs());
- // verify holistic count distinct as a dependent measure
- if (func.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
- throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
- }
+ func.getMeasureType().validate(func);
}
}
@@ -729,9 +727,9 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- public boolean hasHolisticCountDistinctMeasures() {
+ public boolean hasMemoryHungryMeasures() {
for (MeasureDesc measure : measures) {
- if (measure.getFunction().isHolisticCountDistinct()) {
+ if (measure.getFunction().getMeasureType().isMemoryHungry()) {
return true;
}
}
@@ -818,7 +816,8 @@ public class CubeDesc extends RootPersistentEntity {
}
for (MeasureDesc measure : measures) {
- result.addAll(measure.getColumnsNeedDictionary());
+ MeasureType<?> aggrType = measure.getFunction().getMeasureType();
+ result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7561d80c/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 530ce07..5a3202d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -118,10 +118,6 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
if (rtype.isNumberFamily() == false) {
context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
}
- } else if (funcDesc.isTopN()) {
- if (rtype.isTopN() == false) {
- context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of topn");
- }
} else {
if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) {
context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]");
http://git-wip-us.apache.org/repos/asf/kylin/blob/7561d80c/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
index 81b277d..003cb6c 100644
--- a/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
+++ b/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
@@ -127,7 +127,7 @@ public class DataModelDescUpgradeV2 {
if (!modelDimMap.containsKey(cubeDimDesc.getTable())) {
modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>());
}
- modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn()));
+ modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : new String[] { cubeDimDesc.getColumn() }));
}
List<DimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7561d80c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 2f78a86..93d7d6e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -30,8 +30,8 @@ import org.apache.kylin.gridtable.GTScanRangePlanner;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
[03/21] kylin git commit: KYLIN-976 very initial
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 18884f7..be92f73 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -2,9 +2,9 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
@SuppressWarnings({ "rawtypes", "unchecked" })
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 48bda9f..0e61cf2 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -2,8 +2,8 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
public interface IGTCodeSystem {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index 1a9f637..9852acc 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.DataType;
public class UnitTestSupport {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
new file mode 100644
index 0000000..27aa07c
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.aggregation.basic;
+
+import static org.junit.Assert.assertEquals;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.basic.BigDecimalSerializer;
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ */
+public class BigDecimalSerializerTest {
+
+ private static BigDecimalSerializer bigDecimalSerializer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ bigDecimalSerializer = new BigDecimalSerializer(DataType.getInstance("decimal"));
+ }
+
+ @Test
+ public void testNormal() {
+ BigDecimal input = new BigDecimal("1234.1234");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testScaleOutOfRange() {
+ BigDecimal input = new BigDecimal("1234.1234567890");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutOfPrecision() {
+ BigDecimal input = new BigDecimal("66855344214907231736.4924");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ bigDecimalSerializer.serialize(input, buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
new file mode 100644
index 0000000..8ee7b8d
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
@@ -0,0 +1,61 @@
+package org.apache.kylin.aggregation.topn;
+
+import org.apache.kylin.aggregation.topn.TopNCounterSerializer;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+public class TopNCounterSerializerTest {
+
+ private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getInstance("topn(10)"));
+
+ @Test
+ public void testSerialization() {
+ TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
+ Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
+ for (Integer i : stream) {
+ vs.offer(new ByteArray(Bytes.toBytes(i)));
+ }
+
+ ByteBuffer out = ByteBuffer.allocate(1024);
+ serializer.serialize(vs, out);
+
+ byte[] copyBytes = new byte[out.position()];
+ System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
+
+ ByteBuffer in = ByteBuffer.wrap(copyBytes);
+ TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
+
+ Assert.assertEquals(vs.toString(), vsNew.toString());
+
+ }
+
+ @Test
+ public void testValueOf() {
+
+ TopNCounter<ByteArray> origin = new TopNCounter<ByteArray>(10);
+ ByteArray key = new ByteArray(1);
+ ByteBuffer byteBuffer = key.asBuffer();
+ BytesUtil.writeVLong(20l, byteBuffer);
+ origin.offer(key, 1.0);
+
+ byteBuffer = ByteBuffer.allocate(1024);
+ byteBuffer.putInt(1);
+ byteBuffer.putInt(20);
+ byteBuffer.putDouble(1.0);
+ TopNCounter<ByteArray> counter = serializer.valueOf(byteBuffer.array());
+
+
+ Assert.assertEquals(origin.toString(), counter.toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fa7d611..f407977 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -23,15 +23,15 @@ import java.util.Random;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
+import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
+import org.apache.kylin.aggregation.basic.LongSumAggregator;
+import org.apache.kylin.aggregation.hllc.HLLCAggregator;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.measure.BigDecimalSumAggregator;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.measure.DoubleSumAggregator;
-import org.apache.kylin.metadata.measure.HLLCAggregator;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.LongSumAggregator;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.junit.Test;
public class AggregationCacheMemSizeTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 91e7e18..f5247e2 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.Dictionary;
@@ -44,7 +45,6 @@ import org.apache.kylin.metadata.filter.ExtractTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 6395a3b..361617a 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -27,8 +27,8 @@ import java.util.BitSet;
import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.junit.Test;
public class SimpleGridTableTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 41d75c9..a964e67 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -23,20 +23,21 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import org.apache.kylin.aggregation.basic.StringSerializer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.serializer.StringSerializer;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.Test;
import com.google.common.collect.Lists;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
public class SimpleInvertedIndexTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index ad5048b..0e13d7e 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -23,7 +23,10 @@ import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
deleted file mode 100644
index 91713a4..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) < 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
deleted file mode 100644
index 6bbb6a9..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
deleted file mode 100644
index f1c60a1..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal sum = new BigDecimal(0);
-
- @Override
- public void reset() {
- sum = new BigDecimal(0);
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- sum = sum.add(value);
- }
-
- @Override
- public BigDecimal getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
deleted file mode 100644
index 962cd75..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
deleted file mode 100644
index f8c3680..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
deleted file mode 100644
index 10072a6..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
-
- private double v;
-
- public DoubleMutable() {
- this(0);
- }
-
- public DoubleMutable(double v) {
- set(v);
- }
-
- public double get() {
- return v;
- }
-
- public void set(double v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DoubleMutable)) {
- return false;
- }
- DoubleMutable other = (DoubleMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) Double.doubleToLongBits(v);
- }
-
- @Override
- public int compareTo(DoubleMutable o) {
- return (v < o.v ? -1 : (v == o.v ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Double.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
deleted file mode 100644
index 7593cac..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable sum = new DoubleMutable();
-
- @Override
- public void reset() {
- sum.set(0.0);
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
deleted file mode 100644
index f76cf64..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-
-/**
- */
-@SuppressWarnings("serial")
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
-
- final int precision;
- HyperLogLogPlusCounter sum = null;
-
- public HLLCAggregator(int precision) {
- this.precision = precision;
- }
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(HyperLogLogPlusCounter value) {
- if (sum == null)
- sum = new HyperLogLogPlusCounter(value);
- else
- sum.merge(value);
- }
-
- @Override
- public HyperLogLogPlusCounter getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- // 1024 + 60 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 4 // precision
- + 8 // ref to HLLC
- + 8 // HLLC obj shell
- + 32 + (1 << precision); // HLLC internal
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
deleted file mode 100644
index e5cf2a9..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- * Long Distinct Count
- */
-@SuppressWarnings("serial")
-public class LDCAggregator extends MeasureAggregator<LongMutable> {
-
- private static LongMutable ZERO = new LongMutable(0);
-
- private HLLCAggregator hllAgg = null;
- private LongMutable state = new LongMutable(0);
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- this.hllAgg = (HLLCAggregator) agg;
- }
-
- @Override
- public void reset() {
- }
-
- @Override
- public void aggregate(LongMutable value) {
- }
-
- @Override
- public LongMutable getState() {
- if (hllAgg == null) {
- return ZERO;
- } else {
- state.set(hllAgg.getState().getCountEstimate());
- return state;
- }
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
deleted file mode 100644
index 5b080bb..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
deleted file mode 100644
index a2509ac..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
deleted file mode 100644
index 220b6a6..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class LongMutable implements Comparable<LongMutable>, Serializable {
-
- private long v;
-
- public LongMutable() {
- this(0);
- }
-
- public LongMutable(long v) {
- set(v);
- }
-
- public long get() {
- return v;
- }
-
- public void set(long v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof LongMutable)) {
- return false;
- }
- LongMutable other = (LongMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) v;
- }
-
- @Override
- public int compareTo(LongMutable o) {
- long thisValue = this.v;
- long thatValue = o.v;
- return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Long.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
deleted file mode 100644
index 63422f5..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable sum = new LongMutable();
-
- @Override
- public void reset() {
- sum.set(0);
- }
-
- @Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public LongMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
deleted file mode 100644
index 1b38aa5..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-import java.io.Serializable;
-
-/**
- */
-@SuppressWarnings("serial")
-abstract public class MeasureAggregator<V> implements Serializable {
-
- public static MeasureAggregator<?> create(String funcName, String returnType) {
- if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongSumAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalSumAggregator();
- else if (isDouble(returnType))
- return new DoubleSumAggregator();
- } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
- DataType hllcType = DataType.getInstance(returnType);
- if (hllcType.isHLLC())
- return new HLLCAggregator(hllcType.getPrecision());
- else
- return new LDCAggregator();
- } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMaxAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMaxAggregator();
- else if (isDouble(returnType))
- return new DoubleMaxAggregator();
- } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMinAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMinAggregator();
- else if (isDouble(returnType))
- return new DoubleMinAggregator();
- } else if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName)) {
- return new TopNAggregator();
- }
- throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'");
- }
-
- public static boolean isBigDecimal(String type) {
- return type.startsWith("decimal");
- }
-
- public static boolean isDouble(String type) {
- return "double".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "real".equalsIgnoreCase(type);
- }
-
- public static boolean isInteger(String type) {
- return "long".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type) || "integer".equalsIgnoreCase(type);
- }
-
- public static int guessBigDecimalMemBytes() {
- // 116 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 8 // ref to BigDecimal
- + 8 // BigDecimal obj shell
- + 100; // guess of BigDecimal internal
- }
-
- public static int guessDoubleMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to DoubleWritable
- + 8 // DoubleWritable obj shell
- + 8; // size of double
- */
- }
-
- public static int guessLongMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to LongWritable
- + 8 // LongWritable obj shell
- + 8; // size of long
- */
- }
-
- // ============================================================================
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- }
-
- abstract public void reset();
-
- abstract public void aggregate(V value);
-
- abstract public V getState();
-
- // get an estimate of memory consumption UPPER BOUND
- abstract public int getMemBytesEstimate();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
deleted file mode 100644
index 09e553e..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- */
-@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
-public class MeasureAggregators implements Serializable {
-
- private final MeasureAggregator[] aggs;
- private final int descLength;
-
- public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureAggregators(MeasureDesc... measureDescs) {
- descLength = measureDescs.length;
- aggs = new MeasureAggregator[descLength];
-
- Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
- for (int i = 0; i < descLength; i++) {
- FunctionDesc func = measureDescs[i].getFunction();
- aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
- measureIndexMap.put(measureDescs[i].getName(), i);
- }
- // fill back dependent aggregator
- for (int i = 0; i < descLength; i++) {
- String depMsrRef = measureDescs[i].getDependentMeasureRef();
- if (depMsrRef != null) {
- int index = measureIndexMap.get(depMsrRef);
- aggs[i].setDependentAggregator(aggs[index]);
- }
- }
- }
-
- public void reset() {
- for (int i = 0; i < aggs.length; i++) {
- aggs[i].reset();
- }
- }
-
- public void aggregate(Object[] values) {
- assert values.length == descLength;
-
- for (int i = 0; i < descLength; i++) {
- aggs[i].aggregate(values[i]);
- }
- }
-
- public void collectStates(Object[] states) {
- for (int i = 0; i < descLength; i++) {
- states[i] = aggs[i].getState();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
deleted file mode 100644
index 296290a..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MeasureCodec {
-
- int nMeasures;
- DataTypeSerializer[] serializers;
-
- public MeasureCodec(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureCodec(MeasureDesc... measureDescs) {
- String[] dataTypes = new String[measureDescs.length];
- for (int i = 0; i < dataTypes.length; i++) {
- dataTypes[i] = measureDescs[i].getFunction().getReturnType();
- }
- init(dataTypes);
- }
-
- public MeasureCodec(String... dataTypes) {
- init(dataTypes);
- }
-
- private void init(String[] dataTypes) {
- nMeasures = dataTypes.length;
- serializers = new DataTypeSerializer[nMeasures];
-
- for (int i = 0; i < nMeasures; i++) {
- serializers[i] = DataTypeSerializer.create(dataTypes[i]);
- }
- }
-
- public DataTypeSerializer getSerializer(int idx) {
- return serializers[idx];
- }
-
- public void decode(ByteBuffer buf, Object[] result) {
- assert result.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- result[i] = serializers[i].deserialize(buf);
- }
- }
-
- public void encode(Object[] values, ByteBuffer out) {
- assert values.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- serializers[i].serialize(values[i], out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
deleted file mode 100644
index 9c21f7b..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
-
- int capacity = 0;
- TopNCounter<ByteArray> sum = null;
- Map<ByteArray, Double> sanityCheckMap;
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(TopNCounter<ByteArray> value) {
- if (sum == null) {
- capacity = value.getCapacity();
- sum = new TopNCounter<>(capacity);
- sanityCheckMap = Maps.newHashMap();
- }
- sum.merge(value);
- }
-
- @Override
- public TopNCounter<ByteArray> getState() {
-
- //sum.retain(capacity);
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return 8 * capacity / 4;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
deleted file mode 100644
index a0af6a2..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
-
- private DataType type;
- private int presision;
- private HyperLogLogPlusCounter current;
-
- public FixedHLLCodec(DataType type) {
- this.type = type;
- this.presision = type.getPrecision();
- this.current = new HyperLogLogPlusCounter(this.presision);
- }
-
- @Override
- public int getLength() {
- return 1 << presision;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(String value) {
- current.clear();
- if (value == null)
- current.add("__nUlL__");
- else
- current.add(value.getBytes());
- return current;
- }
-
- @Override
- public Object getValue() {
- return current;
- }
-
- @Override
- public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- return current;
- }
-
- @Override
- public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
- v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- }
-
- @Override
- public HyperLogLogPlusCounter read(ByteBuffer buffer) {
- current.readRegistersArray(buffer);
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
deleted file mode 100644
index 54c4eb8..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-
-abstract public class FixedLenMeasureCodec<T> {
-
- public static FixedLenMeasureCodec<?> get(DataType type) {
- if (type.isHLLC()) {
- return new FixedHLLCodec(type);
- } else {
- return new FixedPointLongCodec(type);
- }
- }
-
- abstract public int getLength();
-
- abstract public DataType getDataType();
-
- abstract public T valueOf(String value);
-
- abstract public Object getValue();
-
- abstract public T read(byte[] buf, int offset);
-
- abstract public void write(T v, byte[] buf, int offset);
-
- abstract public T read(ByteBuffer buffer);
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
deleted file mode 100644
index 37eb0be..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-
-public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
-
- private static final int SIZE = 8;
- // number of digits after decimal point
- int scale;
- DataType type;
- // avoid massive object creation
- LongMutable current = new LongMutable();
-
- public FixedPointLongCodec(DataType type) {
- this.type = type;
- this.scale = Math.max(0, type.getScale());
- }
-
- @Override
- public int getLength() {
- return SIZE;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- long getValueIgnoringDecimalPoint(String value) {
- int index = value.indexOf('.');
-
- if (index == 0 || index == value.length() - 1) {
- throw new RuntimeException("Bad decimal format: " + value);
- } else if (index < 0) {
- return Long.valueOf(value) * (int) Math.pow(10, scale);
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append(value.substring(0, index));
-
- //if there are more than scale digits after the decimal point, the tail will be discarded
- int end = Math.min(value.length(), index + scale + 1);
- sb.append(value.substring(index + 1, end));
- int diff = index + scale + 1 - value.length();
- //if there are less than scale digits after the decimal point, the tail will be compensated
- for (int i = 0; i < diff; i++) {
- sb.append('0');
- }
- return Long.valueOf(sb.toString());
- }
- }
-
- String restoreDecimalPoint(long value) {
- if (scale < 0) {
- throw new RuntimeException("Bad scale: " + scale + " with value: " + value);
- } else if (scale == 0) {
- return Long.toString(value);
- } else {
- return String.format("%." + scale + "f", value / (Math.pow(10, scale)));
- }
- }
-
- @Override
- public LongMutable valueOf(String value) {
- if (value == null)
- current.set(0L);
- else
- current.set(getValueIgnoringDecimalPoint(value));
- return current;
- }
-
- @Override
- public String getValue() {
- if (scale == 0)
- return current.toString();
- else
- return restoreDecimalPoint(current.get());
- }
-
- @Override
- public LongMutable read(byte[] buf, int offset) {
- current.set(BytesUtil.readLong(buf, offset, SIZE));
- return current;
- }
-
- @Override
- public void write(LongMutable v, byte[] buf, int offset) {
- BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE);
- }
-
- @Override
- public LongMutable read(ByteBuffer buffer) {
- current.set(BytesUtil.readLong(buffer, SIZE));
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
deleted file mode 100644
index d10c565..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
-
- private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
-
- final DataType type;
- final int maxLength;
-
- int avoidVerbose = 0;
-
- public BigDecimalSerializer(DataType type) {
- this.type = type;
- // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
- this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
- }
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- if (value.scale() > type.getScale()) {
- if (avoidVerbose % 10000 == 0) {
- logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
- }
- value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
- }
- byte[] bytes = value.unscaledValue().toByteArray();
- if (bytes.length + 2 > maxLength) {
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- @SuppressWarnings("unused")
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
- int len = in.position() - mark + n;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
deleted file mode 100644
index d542098..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- *
- * Note: the implementations MUST be thread-safe.
- *
- */
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
-
- final static Map<String, Class<?>> implementations;
- static {
- HashMap<String, Class<?>> impl = Maps.newHashMap();
- impl.put("varchar", StringSerializer.class);
- impl.put("decimal", BigDecimalSerializer.class);
- impl.put("double", DoubleSerializer.class);
- impl.put("float", DoubleSerializer.class);
- impl.put("bigint", LongSerializer.class);
- impl.put("long", LongSerializer.class);
- impl.put("integer", LongSerializer.class);
- impl.put("int", LongSerializer.class);
- impl.put("smallint", LongSerializer.class);
- impl.put("date", DateTimeSerializer.class);
- impl.put("datetime", DateTimeSerializer.class);
- impl.put("timestamp", DateTimeSerializer.class);
- implementations = Collections.unmodifiableMap(impl);
-
- }
-
- public static DataTypeSerializer<?> create(String dataType) {
- return create(DataType.getInstance(dataType));
- }
-
- public static DataTypeSerializer<?> create(DataType type) {
- if (type.isHLLC()) {
- return new HLLCSerializer(type);
- }
-
- if (type.isTopN()) {
- return new TopNCounterSerializer(type);
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + type);
-
- try {
- return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** peek into buffer and return the length of serialization */
- abstract public int peekLength(ByteBuffer in);
-
- /** return the max number of bytes to the longest serialization */
- abstract public int maxLength();
-
- /** get an estimate of size in bytes of the serialized data */
- abstract public int getStorageBytesEstimate();
-
- /** convert from String to obj (string often come as byte[] in mapred) */
- abstract public T valueOf(byte[] value);
-
- /** convert from String to obj */
- public T valueOf(String value) {
- try {
- return valueOf(value.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** convert from obj to string */
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
deleted file mode 100644
index d79c382..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public DateTimeSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(DateFormat.stringToMillis(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
deleted file mode 100644
index 2dc8a67..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
-
- public DoubleSerializer(DataType type) {
- }
-
- @Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
- }
-
- @Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public DoubleMutable valueOf(byte[] value) {
- DoubleMutable d = current();
- if (value == null)
- d.set(0d);
- else
- d.set(Double.parseDouble(Bytes.toString(value)));
- return d;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
deleted file mode 100644
index 7aed458..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
-
- private int precision;
-
- public HLLCSerializer(DataType type) {
- this.precision = type.getPrecision();
- }
-
- @Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
- try {
- value.writeRegisters(out);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private HyperLogLogPlusCounter current() {
- HyperLogLogPlusCounter hllc = current.get();
- if (hllc == null) {
- hllc = new HyperLogLogPlusCounter(precision);
- current.set(hllc);
- }
- return hllc;
- }
-
- @Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- HyperLogLogPlusCounter hllc = current();
- try {
- hllc.readRegisters(in);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return hllc;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return current().peekLength(in);
- }
-
- @Override
- public int maxLength() {
- return current().maxLength();
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return current().maxLength();
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(byte[] value) {
- HyperLogLogPlusCounter hllc = current();
- hllc.clear();
- if (value == null)
- hllc.add("__nUlL__");
- else
- hllc.add(value);
- return hllc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
deleted file mode 100644
index f51a52c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public LongSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- BytesUtil.readVLong(in);
- int len = in.position() - mark;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return 9; // vlong: 1 + 8
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 5;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(Long.parseLong(Bytes.toString(value)));
- return l;
- }
-
-}
[16/21] kylin git commit: KYLIN-976 Extract fillTuple() methods
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
index c395fd9..bde8dd2 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.Tuple;
import org.junit.Test;
/**
[07/21] kylin git commit: KYLIN-976 AggregationType interface,
serializer and aggregator
Posted by li...@apache.org.
KYLIN-976 AggregationType interface, serializer and aggregator
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2d82d38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2d82d38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2d82d38
Branch: refs/heads/2.x-staging
Commit: b2d82d3863a9aaa68a3b370d6f0cb598e07b4ce5
Parents: 836f233
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Nov 25 11:13:18 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:48:43 2015 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 2 +-
.../streaming/StreamingTableDataGenerator.java | 2 +-
.../common/datatype/BigDecimalSerializer.java | 109 +++++++
.../apache/kylin/common/datatype/DataType.java | 289 ++++++++++++++++++
.../common/datatype/DataTypeSerializer.java | 108 +++++++
.../common/datatype/DateTimeSerializer.java | 62 ++++
.../kylin/common/datatype/DoubleMutable.java | 68 +++++
.../kylin/common/datatype/DoubleSerializer.java | 81 +++++
.../kylin/common/datatype/LongMutable.java | 70 +++++
.../kylin/common/datatype/LongSerializer.java | 88 ++++++
.../kylin/common/datatype/StringSerializer.java | 54 ++++
.../kylin/common/debug/BackdoorToggles.java | 5 -
.../kylin/common/topn/DoublyLinkedList.java | 2 -
.../apache/kylin/common/util/DoubleMutable.java | 68 -----
.../apache/kylin/common/util/LongMutable.java | 70 -----
.../datatype/BigDecimalSerializerTest.java | 51 ++++
.../apache/kylin/aggregation/Aggregation.java | 42 ---
.../kylin/aggregation/AggregationType.java | 97 ++++++
.../kylin/aggregation/DataTypeSerializer.java | 118 --------
.../kylin/aggregation/IAggregationFactory.java | 6 +
.../kylin/aggregation/MeasureAggregator.java | 61 +---
.../apache/kylin/aggregation/MeasureCodec.java | 1 +
.../aggregation/basic/BasicAggregation.java | 128 ++++++++
.../basic/BasicAggregationFactory.java | 31 ++
.../aggregation/basic/BigDecimalSerializer.java | 111 -------
.../aggregation/basic/DateTimeSerializer.java | 65 ----
.../aggregation/basic/DoubleMaxAggregator.java | 2 +-
.../aggregation/basic/DoubleMinAggregator.java | 2 +-
.../aggregation/basic/DoubleSerializer.java | 84 ------
.../aggregation/basic/DoubleSumAggregator.java | 2 +-
.../aggregation/basic/LongMaxAggregator.java | 2 +-
.../aggregation/basic/LongMinAggregator.java | 2 +-
.../kylin/aggregation/basic/LongSerializer.java | 91 ------
.../aggregation/basic/LongSumAggregator.java | 2 +-
.../aggregation/basic/StringSerializer.java | 56 ----
.../kylin/aggregation/hllc/HLLCAggregation.java | 78 +++++
.../hllc/HLLCAggregationFactory.java | 35 +++
.../kylin/aggregation/hllc/HLLCSerializer.java | 4 +-
.../kylin/aggregation/hllc/LDCAggregator.java | 2 +-
.../kylin/aggregation/topn/TopNAggregation.java | 76 +++++
.../topn/TopNAggregationFactory.java | 35 +++
.../aggregation/topn/TopNCounterSerializer.java | 12 +-
.../kylin/cube/gridtable/CubeCodeSystem.java | 4 +-
.../kylin/cube/gridtable/CubeGridTable.java | 1 -
.../gridtable/CuboidToGridTableMapping.java | 2 +-
.../cube/gridtable/TrimmedCubeCodeSystem.java | 3 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +-
.../InMemCubeBuilderInputConverter.java | 2 +-
.../apache/kylin/cube/kv/RowKeyColumnOrder.java | 2 +-
.../model/validation/rule/FunctionRule.java | 2 +-
.../java/org/apache/kylin/gridtable/GTInfo.java | 2 +-
.../kylin/gridtable/GTSampleCodeSystem.java | 2 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 14 +-
.../basic/BigDecimalSerializerTest.java | 53 ----
.../topn/TopNCounterSerializerTest.java | 9 +-
.../gridtable/AggregationCacheMemSizeTest.java | 4 +-
.../kylin/gridtable/DictGridTableTest.java | 16 +-
.../kylin/gridtable/SimpleGridTableTest.java | 2 +-
.../gridtable/SimpleInvertedIndexTest.java | 13 +-
.../metadata/measure/MeasureCodecTest.java | 4 +-
.../apache/kylin/dict/DictionaryGenerator.java | 9 +-
.../apache/kylin/dict/DictionaryManager.java | 4 +-
.../apache/kylin/dict/NumberDictionaryTest.java | 9 +-
.../filter/TimeConditionLiteralsReplacer.java | 2 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 9 +-
.../apache/kylin/metadata/model/DataType.java | 295 -------------------
.../kylin/metadata/model/FunctionDesc.java | 11 +-
.../apache/kylin/metadata/model/TblColRef.java | 5 +-
.../metadata/realization/SQLDigestUtil.java | 6 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 4 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 2 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 15 +-
.../invertedindex/index/RawTableRecord.java | 2 +-
.../kylin/invertedindex/index/TableRecord.java | 2 +-
.../invertedindex/index/TableRecordInfo.java | 6 +-
.../index/TableRecordInfoDigest.java | 6 +-
.../invertedindex/measure/FixedHLLCodec.java | 2 +-
.../measure/FixedLenMeasureCodec.java | 4 +-
.../measure/FixedPointLongCodec.java | 4 +-
.../invertedindex/model/IIKeyValueCodec.java | 6 +-
.../measure/FixedPointLongCodecTest.java | 13 +-
.../endpoint/EndpointAggregators.java | 6 +-
.../storage/hbase/steps/CreateHTableJob.java | 2 +-
.../storage/hbase/steps/RowValueDecoder.java | 4 +-
.../observer/AggregateRegionObserverTest.java | 2 +-
.../endpoint/EndpointAggregationTest.java | 2 +-
.../hbase/steps/RowValueDecoderTest.java | 2 +-
87 files changed, 1601 insertions(+), 1244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index c033117..896cb63 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -37,6 +37,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.CubeInstance;
@@ -45,7 +46,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
index 075a048..04a5b5b 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
@@ -5,9 +5,9 @@ import java.util.List;
import java.util.Random;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
new file mode 100644
index 0000000..f19ce14
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author yangli9
+ *
+ */
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
+
+ private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
+
+ final DataType type;
+ final int maxLength;
+
+ int avoidVerbose = 0;
+
+ public BigDecimalSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
+ this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
+ }
+
+ @Override
+ public void serialize(BigDecimal value, ByteBuffer out) {
+ if (value.scale() > type.getScale()) {
+ if (avoidVerbose % 10000 == 0) {
+ logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
+ }
+ value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
+ }
+ byte[] bytes = value.unscaledValue().toByteArray();
+ if (bytes.length + 2 > maxLength) {
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ BytesUtil.writeVInt(value.scale(), out);
+ BytesUtil.writeVInt(bytes.length, out);
+ out.put(bytes);
+ }
+
+ @Override
+ public BigDecimal deserialize(ByteBuffer in) {
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+
+ byte[] bytes = new byte[n];
+ in.get(bytes);
+
+ return new BigDecimal(new BigInteger(bytes), scale);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public BigDecimal valueOf(byte[] value) {
+ if (value == null)
+ return new BigDecimal(0);
+ else
+ return new BigDecimal(Bytes.toString(value));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
new file mode 100644
index 0000000..36fad00
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DataType implements Serializable {
+
+ // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
+ public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
+ + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
+ + "|date|time|datetime|timestamp|byte|int|short|long|string";
+
+ private static final String TYPE_PATTEN_TAIL = "\\s*" //
+ + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";
+
+ private static final Pattern TYPE_PATTERN = Pattern.compile( //
+ "(" + VALID_TYPES_STRING + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
+
+ private static final Pattern CUSTOM_TYPE_PATTERN = Pattern.compile( //
+ "(" + ".*?" + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
+
+ public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
+ public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
+ public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
+ public static final Set<String> STRING_FAMILY = new HashSet<String>();
+ private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
+ static {
+ INTEGER_FAMILY.add("tinyint");
+ INTEGER_FAMILY.add("smallint");
+ INTEGER_FAMILY.add("integer");
+ INTEGER_FAMILY.add("bigint");
+
+ NUMBER_FAMILY.addAll(INTEGER_FAMILY);
+ NUMBER_FAMILY.add("float");
+ NUMBER_FAMILY.add("double");
+ NUMBER_FAMILY.add("decimal");
+ NUMBER_FAMILY.add("real");
+ NUMBER_FAMILY.add("numeric");
+
+ DATETIME_FAMILY.add("date");
+ DATETIME_FAMILY.add("time");
+ DATETIME_FAMILY.add("datetime");
+ DATETIME_FAMILY.add("timestamp");
+
+ STRING_FAMILY.add("varchar");
+ STRING_FAMILY.add("char");
+
+ LEGACY_TYPE_MAP.put("byte", "tinyint");
+ LEGACY_TYPE_MAP.put("int", "integer");
+ LEGACY_TYPE_MAP.put("short", "smallint");
+ LEGACY_TYPE_MAP.put("long", "bigint");
+ LEGACY_TYPE_MAP.put("string", "varchar");
+ LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
+ LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
+ LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
+ LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
+ LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
+ }
+
+ private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
+
+ public static final DataType ANY = DataType.getType("any");
+
+ public static DataType getType(String type) {
+ return getTypeInner(type, false);
+ }
+
+ public static DataType getCustomType(String type) {
+ return getTypeInner(type, true);
+ }
+
+ private static DataType getTypeInner(String type, boolean isCustom) {
+ if (type == null)
+ return null;
+
+ DataType dataType = new DataType(type, isCustom);
+ DataType cached = CACHE.get(dataType);
+ if (cached == null) {
+ CACHE.put(dataType, dataType);
+ cached = dataType;
+ }
+ return cached;
+ }
+
+ // ============================================================================
+
+ private String name;
+ private int precision;
+ private int scale;
+
+ DataType(String datatype, boolean isCustom) {
+ datatype = datatype.trim().toLowerCase();
+ datatype = replaceLegacy(datatype);
+
+ Pattern pattern = isCustom ? CUSTOM_TYPE_PATTERN : TYPE_PATTERN;
+ Matcher m = pattern.matcher(datatype);
+ if (m.matches() == false)
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", does not match " + pattern);
+
+ name = replaceLegacy(m.group(1));
+ precision = -1;
+ scale = -1;
+
+ String leftover = m.group(2);
+ if (leftover != null) {
+ String[] parts = leftover.split("\\s*,\\s*");
+ for (int i = 0; i < parts.length; i++) {
+ int n;
+ try {
+ n = Integer.parseInt(parts[i]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric");
+ }
+ if (i == 0)
+ precision = n;
+ else if (i == 1)
+ scale = n;
+ else
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts");
+ }
+ }
+
+ // FIXME 256 for unknown string precision
+ if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
+ precision = 256; // to save memory at frontend, e.g. tableau will
+ // allocate memory according to this
+ }
+
+ // FIXME (19,4) for unknown decimal precision
+ if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) {
+ precision = 19;
+ scale = 4;
+ }
+ }
+
+ private String replaceLegacy(String str) {
+ String replace = LEGACY_TYPE_MAP.get(str);
+ return replace == null ? str : replace;
+ }
+
+ public int getStorageBytesEstimate() {
+ return DataTypeSerializer.create(this).getStorageBytesEstimate();
+ }
+
+ public boolean isStringFamily() {
+ return STRING_FAMILY.contains(name);
+ }
+
+ public boolean isIntegerFamily() {
+ return INTEGER_FAMILY.contains(name);
+ }
+
+ public boolean isNumberFamily() {
+ return NUMBER_FAMILY.contains(name);
+ }
+
+ public boolean isDateTimeFamily() {
+ return DATETIME_FAMILY.contains(name);
+ }
+
+ public boolean isDate() {
+ return name.equals("date");
+ }
+
+ public boolean isTime() {
+ return name.equals("time");
+ }
+
+ public boolean isTimestamp() {
+ return name.equals("timestamp");
+ }
+
+ public boolean isDatetime() {
+ return name.equals("datetime");
+ }
+
+ public boolean isTinyInt() {
+ return name.equals("tinyint");
+ }
+
+ public boolean isSmallInt() {
+ return name.equals("smallint");
+ }
+
+ public boolean isInt() {
+ return name.equals("integer");
+ }
+
+ public boolean isBigInt() {
+ return name.equals("bigint");
+ }
+
+ public boolean isFloat() {
+ return name.equals("float");
+ }
+
+ public boolean isDouble() {
+ return name.equals("double");
+ }
+
+ public boolean isDecimal() {
+ return name.equals("decimal");
+ }
+
+ public boolean isHLLC() {
+ return name.equals("hllc");
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + precision;
+ result = prime * result + scale;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ DataType other = (DataType) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (precision != other.precision)
+ return false;
+ if (scale != other.scale)
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ if (precision < 0 && scale < 0)
+ return name;
+ else if (scale < 0)
+ return name + "(" + precision + ")";
+ else
+ return name + "(" + precision + "," + scale + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
new file mode 100644
index 0000000..99fa3fd
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.common.util.BytesSerializer;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @author yangli9
+ *
+ * Note: the implementations MUST be thread-safe.
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static Map<String, Class<?>> implementations;
+ static {
+ HashMap<String, Class<?>> impl = Maps.newHashMap();
+ impl.put("varchar", StringSerializer.class);
+ impl.put("decimal", BigDecimalSerializer.class);
+ impl.put("double", DoubleSerializer.class);
+ impl.put("float", DoubleSerializer.class);
+ impl.put("bigint", LongSerializer.class);
+ impl.put("long", LongSerializer.class);
+ impl.put("integer", LongSerializer.class);
+ impl.put("int", LongSerializer.class);
+ impl.put("smallint", LongSerializer.class);
+ impl.put("date", DateTimeSerializer.class);
+ impl.put("datetime", DateTimeSerializer.class);
+ impl.put("timestamp", DateTimeSerializer.class);
+ implementations = Collections.unmodifiableMap(impl);
+ }
+
+ public static boolean hasRegistered(String dataTypeName) {
+ return implementations.containsKey(dataTypeName);
+ }
+
+ public static void register(String dataTypeName, Class<? extends DataTypeSerializer<?>> impl) {
+ implementations.put(dataTypeName, impl);
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ return create(DataType.getType(dataType));
+ }
+
+ public static DataTypeSerializer<?> create(DataType type) {
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No DataTypeSerializer for type " + type);
+
+ try {
+ return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** return the max number of bytes to the longest serialization */
+ abstract public int maxLength();
+
+ /** get an estimate of size in bytes of the serialized data */
+ abstract public int getStorageBytesEstimate();
+
+ /** convert from String to obj (string often come as byte[] in mapred) */
+ abstract public T valueOf(byte[] value);
+
+ /** convert from String to obj */
+ public T valueOf(String value) {
+ try {
+ return valueOf(value.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
new file mode 100644
index 0000000..d3ef7cd
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
@@ -0,0 +1,62 @@
+package org.apache.kylin.common.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.DateFormat;
+
+public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public DateTimeSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ out.putLong(value.get());
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(in.getLong());
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public LongMutable valueOf(byte[] value) {
+ LongMutable l = current();
+ if (value == null)
+ l.set(0L);
+ else
+ l.set(DateFormat.stringToMillis(Bytes.toString(value)));
+ return l;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
new file mode 100644
index 0000000..a32fbd8
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
+
+ private double v;
+
+ public DoubleMutable() {
+ this(0);
+ }
+
+ public DoubleMutable(double v) {
+ set(v);
+ }
+
+ public double get() {
+ return v;
+ }
+
+ public void set(double v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DoubleMutable)) {
+ return false;
+ }
+ DoubleMutable other = (DoubleMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) Double.doubleToLongBits(v);
+ }
+
+ @Override
+ public int compareTo(DoubleMutable o) {
+ return (v < o.v ? -1 : (v == o.v ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Double.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
new file mode 100644
index 0000000..f128576
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+
+/**
+ */
+public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
+
+ public DoubleSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(DoubleMutable value, ByteBuffer out) {
+ out.putDouble(value.get());
+ }
+
+ private DoubleMutable current() {
+ DoubleMutable d = current.get();
+ if (d == null) {
+ d = new DoubleMutable();
+ current.set(d);
+ }
+ return d;
+ }
+
+ @Override
+ public DoubleMutable deserialize(ByteBuffer in) {
+ DoubleMutable d = current();
+ d.set(in.getDouble());
+ return d;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public DoubleMutable valueOf(byte[] value) {
+ DoubleMutable d = current();
+ if (value == null)
+ d.set(0d);
+ else
+ d.set(Double.parseDouble(Bytes.toString(value)));
+ return d;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java b/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
new file mode 100644
index 0000000..72b540c
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class LongMutable implements Comparable<LongMutable>, Serializable {
+
+ private long v;
+
+ public LongMutable() {
+ this(0);
+ }
+
+ public LongMutable(long v) {
+ set(v);
+ }
+
+ public long get() {
+ return v;
+ }
+
+ public void set(long v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof LongMutable)) {
+ return false;
+ }
+ LongMutable other = (LongMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) v;
+ }
+
+ @Override
+ public int compareTo(LongMutable o) {
+ long thisValue = this.v;
+ long thatValue = o.v;
+ return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Long.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
new file mode 100644
index 0000000..5c5e10f
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ */
+public class LongSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public LongSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ BytesUtil.writeVLong(value.get(), out);
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(BytesUtil.readVLong(in));
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return 9; // vlong: 1 + 8
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 5;
+ }
+
+ @Override
+ public LongMutable valueOf(byte[] value) {
+ LongMutable l = current();
+ if (value == null)
+ l.set(0L);
+ else
+ l.set(Long.parseLong(Bytes.toString(value)));
+ return l;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
new file mode 100644
index 0000000..b3562eb
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
@@ -0,0 +1,54 @@
+package org.apache.kylin.common.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+
+public class StringSerializer extends DataTypeSerializer<String> {
+
+ final DataType type;
+ final int maxLength;
+
+ public StringSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 2 byte length, rest is String.toBytes()
+ this.maxLength = 2 + type.getPrecision();
+ }
+
+ @Override
+ public void serialize(String value, ByteBuffer out) {
+ int start = out.position();
+
+ BytesUtil.writeUTFString(value, out);
+
+ if (out.position() - start > maxLength)
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ @Override
+ public String deserialize(ByteBuffer in) {
+ return BytesUtil.readUTFString(in);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return BytesUtil.peekByteArrayLength(in);
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return maxLength;
+ }
+
+ @Override
+ public String valueOf(byte[] value) {
+ return Bytes.toString(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 7304b8b..f3745d7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,15 +20,10 @@ package org.apache.kylin.common.debug;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
*/
public class BackdoorToggles {
- private static final Logger logger = LoggerFactory.getLogger(BackdoorToggles.class);
-
private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
public static void setToggles(Map<String, String> toggles) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java b/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
index 1520ce1..4a3d6e8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
@@ -18,8 +18,6 @@
package org.apache.kylin.common.topn;
-import java.util.ConcurrentModificationException;
-import java.util.Iterator;
/**
* Modified from DoublyLinkedList.java in https://github.com/addthis/stream-lib
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java b/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
deleted file mode 100644
index 520cd74..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
-
- private double v;
-
- public DoubleMutable() {
- this(0);
- }
-
- public DoubleMutable(double v) {
- set(v);
- }
-
- public double get() {
- return v;
- }
-
- public void set(double v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DoubleMutable)) {
- return false;
- }
- DoubleMutable other = (DoubleMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) Double.doubleToLongBits(v);
- }
-
- @Override
- public int compareTo(DoubleMutable o) {
- return (v < o.v ? -1 : (v == o.v ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Double.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java b/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
deleted file mode 100644
index 238bb86..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class LongMutable implements Comparable<LongMutable>, Serializable {
-
- private long v;
-
- public LongMutable() {
- this(0);
- }
-
- public LongMutable(long v) {
- set(v);
- }
-
- public long get() {
- return v;
- }
-
- public void set(long v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof LongMutable)) {
- return false;
- }
- LongMutable other = (LongMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) v;
- }
-
- @Override
- public int compareTo(LongMutable o) {
- long thisValue = this.v;
- long thatValue = o.v;
- return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Long.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java b/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
new file mode 100644
index 0000000..cba6795
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
@@ -0,0 +1,51 @@
+package org.apache.kylin.common.datatype;
+
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ */
+public class BigDecimalSerializerTest {
+
+ private static BigDecimalSerializer bigDecimalSerializer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ bigDecimalSerializer = new BigDecimalSerializer(DataType.getType("decimal"));
+ }
+
+ @Test
+ public void testNormal() {
+ BigDecimal input = new BigDecimal("1234.1234");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testScaleOutOfRange() {
+ BigDecimal input = new BigDecimal("1234.1234567890");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutOfPrecision() {
+ BigDecimal input = new BigDecimal("66855344214907231736.4924");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ bigDecimalSerializer.serialize(input, buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
deleted file mode 100644
index 193c5de..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.apache.kylin.aggregation;
-
-import java.util.List;
-
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-abstract public class Aggregation {
-
- /* ============================================================================
- * Define
- * ---------------------------------------------------------------------------- */
-
- abstract public DataType getAggregationDataType();
-
- abstract public DataType getResultDataType();
-
- abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
-
- /* ============================================================================
- * Build
- * ---------------------------------------------------------------------------- */
-
- abstract public DataTypeSerializer<?> getSeralizer();
-
- abstract public MeasureAggregator<?> getAggregator();
-
- abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
-
- abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
-
- /* ============================================================================
- * Cube Selection
- * ---------------------------------------------------------------------------- */
-
- /* ============================================================================
- * Query
- * ---------------------------------------------------------------------------- */
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java b/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
new file mode 100644
index 0000000..81345b7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.aggregation.basic.BasicAggregationFactory;
+import org.apache.kylin.aggregation.hllc.HLLCAggregationFactory;
+import org.apache.kylin.aggregation.topn.TopNAggregationFactory;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Maps;
+
+abstract public class AggregationType {
+
+ private static final Map<String, IAggregationFactory> factoryRegistry = Maps.newConcurrentMap();
+ private static final IAggregationFactory defaultFactory = new BasicAggregationFactory();
+
+ static {
+ factoryRegistry.put(FunctionDesc.FUNC_COUNT_DISTINCT, new HLLCAggregationFactory());
+ factoryRegistry.put(FunctionDesc.FUNC_TOP_N, new TopNAggregationFactory());
+ }
+
+ public static AggregationType create(String funcName, String dataType) {
+ funcName = funcName.toUpperCase();
+ dataType = dataType.toLowerCase();
+
+ IAggregationFactory factory = factoryRegistry.get(funcName);
+ if (factory == null)
+ factory = defaultFactory;
+
+ AggregationType result = factory.createAggregationType(funcName, dataType);
+
+ // register serializer for aggr data type
+ DataType aggregationDataType = result.getAggregationDataType();
+ if (DataTypeSerializer.hasRegistered(aggregationDataType.getName()) == false) {
+ DataTypeSerializer.register(aggregationDataType.getName(), result.getAggregationDataSeralizer());
+ }
+
+ return result;
+ }
+
+ /* ============================================================================
+ * Define
+ * ---------------------------------------------------------------------------- */
+
+ abstract public DataType getAggregationDataType();
+
+ abstract public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer();
+
+ abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
+
+ /* ============================================================================
+ * Build
+ * ---------------------------------------------------------------------------- */
+
+ abstract public MeasureAggregator<?> newAggregator();
+
+ abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
+
+ abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
+
+ /* ============================================================================
+ * Cube Selection
+ * ---------------------------------------------------------------------------- */
+
+ /* ============================================================================
+ * Query
+ * ---------------------------------------------------------------------------- */
+
+ /* ============================================================================
+ * Storage
+ * ---------------------------------------------------------------------------- */
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
deleted file mode 100644
index df6833c..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.aggregation.basic.BigDecimalSerializer;
-import org.apache.kylin.aggregation.basic.DateTimeSerializer;
-import org.apache.kylin.aggregation.basic.DoubleSerializer;
-import org.apache.kylin.aggregation.basic.LongSerializer;
-import org.apache.kylin.aggregation.basic.StringSerializer;
-import org.apache.kylin.aggregation.hllc.HLLCSerializer;
-import org.apache.kylin.aggregation.topn.TopNCounterSerializer;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- *
- * Note: the implementations MUST be thread-safe.
- *
- */
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
-
- final static Map<String, Class<?>> implementations;
- static {
- HashMap<String, Class<?>> impl = Maps.newHashMap();
- impl.put("varchar", StringSerializer.class);
- impl.put("decimal", BigDecimalSerializer.class);
- impl.put("double", DoubleSerializer.class);
- impl.put("float", DoubleSerializer.class);
- impl.put("bigint", LongSerializer.class);
- impl.put("long", LongSerializer.class);
- impl.put("integer", LongSerializer.class);
- impl.put("int", LongSerializer.class);
- impl.put("smallint", LongSerializer.class);
- impl.put("date", DateTimeSerializer.class);
- impl.put("datetime", DateTimeSerializer.class);
- impl.put("timestamp", DateTimeSerializer.class);
- implementations = Collections.unmodifiableMap(impl);
-
- }
-
- public static DataTypeSerializer<?> create(String dataType) {
- return create(DataType.getInstance(dataType));
- }
-
- public static DataTypeSerializer<?> create(DataType type) {
- if (type.isHLLC()) {
- return new HLLCSerializer(type);
- }
-
- if (type.isTopN()) {
- return new TopNCounterSerializer(type);
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + type);
-
- try {
- return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** peek into buffer and return the length of serialization */
- abstract public int peekLength(ByteBuffer in);
-
- /** return the max number of bytes to the longest serialization */
- abstract public int maxLength();
-
- /** get an estimate of size in bytes of the serialized data */
- abstract public int getStorageBytesEstimate();
-
- /** convert from String to obj (string often come as byte[] in mapred) */
- abstract public T valueOf(byte[] value);
-
- /** convert from String to obj */
- public T valueOf(String value) {
- try {
- return valueOf(value.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** convert from obj to string */
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
new file mode 100644
index 0000000..cbdd9da
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
@@ -0,0 +1,6 @@
+package org.apache.kylin.aggregation;
+
+public interface IAggregationFactory {
+
+ public AggregationType createAggregationType(String funcName, String dataType);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
index 9c8945d..6968a9d 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
@@ -18,21 +18,6 @@
package org.apache.kylin.aggregation;
-import org.apache.kylin.aggregation.basic.BigDecimalMaxAggregator;
-import org.apache.kylin.aggregation.basic.BigDecimalMinAggregator;
-import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
-import org.apache.kylin.aggregation.basic.DoubleMaxAggregator;
-import org.apache.kylin.aggregation.basic.DoubleMinAggregator;
-import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
-import org.apache.kylin.aggregation.basic.LongMaxAggregator;
-import org.apache.kylin.aggregation.basic.LongMinAggregator;
-import org.apache.kylin.aggregation.basic.LongSumAggregator;
-import org.apache.kylin.aggregation.hllc.HLLCAggregator;
-import org.apache.kylin.aggregation.hllc.LDCAggregator;
-import org.apache.kylin.aggregation.topn.TopNAggregator;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
import java.io.Serializable;
/**
@@ -40,50 +25,8 @@ import java.io.Serializable;
@SuppressWarnings("serial")
abstract public class MeasureAggregator<V> implements Serializable {
- public static MeasureAggregator<?> create(String funcName, String returnType) {
- if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongSumAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalSumAggregator();
- else if (isDouble(returnType))
- return new DoubleSumAggregator();
- } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
- DataType hllcType = DataType.getInstance(returnType);
- if (hllcType.isHLLC())
- return new HLLCAggregator(hllcType.getPrecision());
- else
- return new LDCAggregator();
- } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMaxAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMaxAggregator();
- else if (isDouble(returnType))
- return new DoubleMaxAggregator();
- } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMinAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMinAggregator();
- else if (isDouble(returnType))
- return new DoubleMinAggregator();
- } else if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName)) {
- return new TopNAggregator();
- }
- throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'");
- }
-
- public static boolean isBigDecimal(String type) {
- return type.startsWith("decimal");
- }
-
- public static boolean isDouble(String type) {
- return "double".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "real".equalsIgnoreCase(type);
- }
-
- public static boolean isInteger(String type) {
- return "long".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type) || "integer".equalsIgnoreCase(type);
+ public static MeasureAggregator<?> create(String funcName, String dataType) {
+ return AggregationType.create(funcName, dataType).newAggregator();
}
public static int guessBigDecimalMemBytes() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
index cbcb3a8..8721954 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
@@ -21,6 +21,7 @@ package org.apache.kylin.aggregation;
import java.nio.ByteBuffer;
import java.util.Collection;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.MeasureDesc;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
new file mode 100644
index 0000000..7ea88bd
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.util.List;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.datatype.BigDecimalSerializer;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
+import org.apache.kylin.common.datatype.DateTimeSerializer;
+import org.apache.kylin.common.datatype.DoubleSerializer;
+import org.apache.kylin.common.datatype.LongSerializer;
+import org.apache.kylin.common.datatype.StringSerializer;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class BasicAggregation extends AggregationType {
+
+ private final String funcName;
+ private final DataType dataType;
+
+ public BasicAggregation(String funcName, String dataType) {
+ this.funcName = funcName;
+ this.dataType = DataType.getType(dataType);
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ if (dataType.isStringFamily())
+ return StringSerializer.class;
+ else if (dataType.isIntegerFamily())
+ return LongSerializer.class;
+ else if (dataType.isDecimal())
+ return BigDecimalSerializer.class;
+ else if (dataType.isNumberFamily())
+ return DoubleSerializer.class;
+ else if (dataType.isDateTimeFamily())
+ return DateTimeSerializer.class;
+ else
+ throw new IllegalArgumentException("No default serializer for type " + dataType);
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ if (isSum() || isCount()) {
+ if (dataType.isDecimal())
+ return new BigDecimalSumAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongSumAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleSumAggregator();
+ } else if (isMax()) {
+ if (dataType.isDecimal())
+ return new BigDecimalMaxAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongMaxAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleMaxAggregator();
+ } else if (isMin()) {
+ if (dataType.isDecimal())
+ return new BigDecimalMinAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongMinAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleMinAggregator();
+ }
+ throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + dataType + "'");
+ }
+
+ private boolean isSum() {
+ return FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isCount() {
+ return FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isMax() {
+ return FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isMin() {
+ return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
new file mode 100644
index 0000000..547d45f
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.IAggregationFactory;
+
+public class BasicAggregationFactory implements IAggregationFactory {
+
+ @Override
+ public AggregationType createAggregationType(String funcName, String dataType) {
+ return new BasicAggregation(funcName, dataType);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
deleted file mode 100644
index 9f7c3cf..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
-
- private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
-
- final DataType type;
- final int maxLength;
-
- int avoidVerbose = 0;
-
- public BigDecimalSerializer(DataType type) {
- this.type = type;
- // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
- this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
- }
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- if (value.scale() > type.getScale()) {
- if (avoidVerbose % 10000 == 0) {
- logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
- }
- value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
- }
- byte[] bytes = value.unscaledValue().toByteArray();
- if (bytes.length + 2 > maxLength) {
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- @SuppressWarnings("unused")
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
- int len = in.position() - mark + n;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
deleted file mode 100644
index 0bf4aba..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.apache.kylin.aggregation.basic;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public DateTimeSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(DateFormat.stringToMillis(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
index f09614d..99896a6 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.datatype.DoubleMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
index b93c15c..7430c4e 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.datatype.DoubleMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
deleted file mode 100644
index f207054..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DoubleMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
-
- public DoubleSerializer(DataType type) {
- }
-
- @Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
- }
-
- @Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public DoubleMutable valueOf(byte[] value) {
- DoubleMutable d = current();
- if (value == null)
- d.set(0d);
- else
- d.set(Double.parseDouble(Bytes.toString(value)));
- return d;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
index 298cec6..6e66c1b 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.datatype.DoubleMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
index 71d95f2..7fdf3d8 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.common.datatype.LongMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
index d1e93f2..22ae865 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.common.datatype.LongMutable;
/**
*/
[20/21] kylin git commit: KYLIN-976 bug fix
Posted by li...@apache.org.
KYLIN-976 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0643f91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0643f91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0643f91
Branch: refs/heads/2.x-staging
Commit: b0643f91cd06d5a051b021add2b6d759abbfa82c
Parents: 7f50c17
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Dec 4 18:24:06 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:55:39 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/CubeCapabilityChecker.java | 2 +-
.../apache/kylin/measure/MeasureTypeFactory.java | 7 +++----
.../apache/kylin/measure/hllc/HLLCMeasureType.java | 15 +++++++++------
.../apache/kylin/measure/topn/TopNMeasureType.java | 17 ++++++++++-------
.../apache/kylin/metadata/datatype/DataType.java | 4 +++-
5 files changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0643f91/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 28252a3..90a3656 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -186,7 +186,7 @@ public class CubeCapabilityChecker {
if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
break;
- MeasureType measureType = measure.getFunction().getMeasureType();
+ MeasureType<?> measureType = measure.getFunction().getMeasureType();
if (measureType instanceof BasicMeasureType)
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0643f91/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 48eddb2..13cc74a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.measure.hllc.HLLCMeasureType;
import org.apache.kylin.measure.topn.TopNMeasureType;
@@ -43,7 +42,7 @@ abstract public class MeasureTypeFactory<T> {
// ============================================================================
- private static Map<Pair<String, String>, MeasureTypeFactory<?>> factories = Maps.newHashMap();
+ private static Map<String, MeasureTypeFactory<?>> factories = Maps.newHashMap();
private static MeasureTypeFactory<?> defaultFactory = new BasicMeasureType.Factory();
public static synchronized void init(KylinConfig config) {
@@ -69,7 +68,7 @@ abstract public class MeasureTypeFactory<T> {
DataType.register(dataTypeName);
DataTypeSerializer.register(dataTypeName, serializer);
- factories.put(Pair.newPair(funcName, dataTypeName), factory);
+ factories.put(funcName, factory);
}
}
@@ -80,7 +79,7 @@ abstract public class MeasureTypeFactory<T> {
public static MeasureType<?> create(String funcName, DataType dataType) {
funcName = funcName.toUpperCase();
- MeasureTypeFactory<?> factory = factories.get(Pair.newPair(funcName, dataType.getName()));
+ MeasureTypeFactory<?> factory = factories.get(funcName);
if (factory == null)
factory = defaultFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0643f91/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 073de6d..4a80d9b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -36,7 +36,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
public static final String DATATYPE_HLLC = "hllc";
-
+
public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounter> {
@Override
@@ -59,7 +59,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
return HLLCSerializer.class;
}
}
-
+
// ============================================================================
private final DataType dataType;
@@ -77,11 +77,14 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
if (FUNC_COUNT_DISTINCT.equals(funcName) == false)
throw new IllegalArgumentException();
- if (DATATYPE_HLLC.equals(dataType.getName()) == false)
- throw new IllegalArgumentException();
+ // data type could be null at query layer, because only function and parameters are known at that stage
+ if (dataType != null) {
+ if (DATATYPE_HLLC.equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
- if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
- throw new IllegalArgumentException();
+ if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
+ throw new IllegalArgumentException();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0643f91/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index c3d36ca..74c6d10 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -53,7 +53,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
public static final String FUNC_TOP_N = "TOP_N";
public static final String DATATYPE_TOPN = "topn";
-
+
public static class Factory extends MeasureTypeFactory<TopNCounter<ByteArray>> {
@Override
@@ -76,7 +76,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return TopNCounterSerializer.class;
}
}
-
+
// ============================================================================
private final DataType dataType;
@@ -93,12 +93,15 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private void validate(String funcName, DataType dataType) {
if (FUNC_TOP_N.equals(funcName) == false)
throw new IllegalArgumentException();
-
- if (DATATYPE_TOPN.equals(dataType.getName()) == false)
- throw new IllegalArgumentException();
- if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
- throw new IllegalArgumentException();
+ // data type could be null at query layer, because only function and parameters are known at that stage
+ if (dataType != null) {
+ if (DATATYPE_TOPN.equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+
+ if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
+ throw new IllegalArgumentException();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0643f91/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 05e6744..f5231e5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -30,6 +30,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
/**
*/
@@ -58,7 +59,8 @@ public class DataType implements Serializable {
"boolean", "byte", "binary", //
"int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
"float", "real", "double", "decimal", "numeric", //
- "date", "time", "datetime", "timestamp");
+ "date", "time", "datetime", "timestamp", //
+ InnerDataTypeEnum.LITERAL.getDataType(), InnerDataTypeEnum.DERIVED.getDataType());
}
public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
[13/21] kylin git commit: KYLIN-976 Extract influenceCapabilityCheck()
Posted by li...@apache.org.
KYLIN-976 Extract influenceCapabilityCheck()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4c4c88a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4c4c88a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4c4c88a
Branch: refs/heads/2.x-staging
Commit: f4c4c88a52a06397c530ea6ec0b33c810b2596f6
Parents: f011d10
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Nov 30 17:55:08 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:52:10 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 215 ++++++++-----------
.../apache/kylin/cube/CubeDimensionDeriver.java | 46 ----
.../org/apache/kylin/cube/CubeInstance.java | 23 +-
.../org/apache/kylin/measure/MeasureType.java | 13 +-
.../kylin/measure/basic/BasicMeasureType.java | 9 -
.../kylin/measure/hllc/HLLCMeasureType.java | 7 -
.../kylin/measure/topn/TopNMeasureType.java | 46 +++-
.../metadata/realization/CapabilityResult.java | 63 ++++++
.../metadata/realization/IRealization.java | 18 +-
.../kylin/storage/hybrid/HybridInstance.java | 32 +--
.../storage/hybrid/HybridStorageQuery.java | 2 +-
.../apache/kylin/invertedindex/IIInstance.java | 29 ++-
.../apache/kylin/query/routing/Candidate.java | 100 +++++++++
.../apache/kylin/query/routing/QueryRouter.java | 56 ++++-
.../apache/kylin/query/routing/RoutingRule.java | 22 +-
.../AdjustForWeaklyMatchedRealization.java | 99 ---------
.../RoutingRules/RealizationPriorityRule.java | 59 -----
.../RoutingRules/RealizationSortRule.java | 65 ------
.../RemoveUncapableRealizationsRule.java | 41 ----
.../routing/rules/RealizationSortRule.java | 34 +++
.../rules/RemoveUncapableRealizationsRule.java | 44 ++++
.../apache/kylin/query/test/ITIIQueryTest.java | 7 +-
22 files changed, 499 insertions(+), 531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 343bf11..0c1b3c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -18,158 +18,104 @@
package org.apache.kylin.cube;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+import com.google.common.collect.Sets;
/**
*/
public class CubeCapabilityChecker {
private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class);
- public static boolean check(CubeInstance cube, SQLDigest digest, boolean allowWeakMatch) {
+ public static CapabilityResult check(CubeInstance cube, SQLDigest digest) {
+ CapabilityResult result = new CapabilityResult();
+ result.capable = false;
// retrieve members from olapContext
- Collection<TblColRef> dimensionColumns = CubeDimensionDeriver.getDimensionColumns(digest);
- Collection<FunctionDesc> functions = digest.aggregations;
- Collection<TblColRef> metricsColumns = digest.metricColumns;
+ Collection<TblColRef> dimensionColumns = getDimensionColumns(digest);
+ Collection<FunctionDesc> aggrFunctions = digest.aggregations;
Collection<JoinDesc> joins = digest.joinDescs;
- boolean hasTopN = hasTopNMeasure(cube.getDescriptor());
// match dimensions & aggregations & joins
- boolean isOnline = cube.isReady();
-
- boolean matchDimensions = isMatchedWithDimensions(dimensionColumns, cube);
- boolean matchAggregation = isMatchedWithAggregations(functions, cube);
- boolean matchJoin = isMatchedWithJoins(joins, cube);
-
- // Some cubes are not "perfectly" match, but still save them in case of usage
- if (allowWeakMatch && isOnline && matchDimensions && !matchAggregation && matchJoin) {
- // sometimes metrics are indeed dimensions
- // e.g. select min(cal_dt) from ..., where cal_dt is actually a dimension
- if (isWeaklyMatchedWithAggregations(functions, metricsColumns, cube)) {
- logger.info("Weakly matched cube found " + cube.getName());
- return true;
- }
- }
+ Collection<TblColRef> unmatchedDimensions = unmatchedDimensions(dimensionColumns, cube);
+ Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube);
+ boolean isJoinMatch = isJoinMatch(joins, cube);
- // for topn, the group column can come from measure
- if (hasTopN & matchJoin && !matchDimensions && functions.size() == 1) {
- boolean matchedTopN = isMatchedWithTopN(dimensionColumns, cube, digest);
- matchDimensions = matchedTopN;
- matchAggregation = matchedTopN;
+ if (!isJoinMatch) {
+ logger.info("Exclude cube " + cube.getName() + " because unmatched joins");
+ return result;
}
- if (!isOnline || !matchDimensions || !matchAggregation || !matchJoin) {
- logger.info("Exclude cube " + cube.getName() + " because " + " isOnlne=" + isOnline + ",matchDimensions=" + matchDimensions + ",matchAggregation=" + matchAggregation + ",matchJoin=" + matchJoin);
- return false;
+ // try custom measure types
+ if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) {
+ tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
}
-
- return true;
- }
-
- /**
- * Check whether the cube can match the sql digest with TopN measure
- * @param cube
- * @param digest
- * @return
- */
- public static boolean isMatchedWithTopN(CubeInstance cube, SQLDigest digest) {
-
- boolean hasTopN = hasTopNMeasure(cube.getDescriptor());
- if (hasTopN == false) {
- return false;
+ // try dimension-as-measure
+ if (!unmatchedAggregations.isEmpty()) {
+ tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result);
}
- Collection<TblColRef> dimensionColumns = CubeDimensionDeriver.getDimensionColumns(digest);
-
- boolean matchDimensions = isMatchedWithDimensions(dimensionColumns, cube);
-
- if (matchDimensions == true) {
- return false;
- }
-
- Collection<FunctionDesc> functions = digest.aggregations;
- if (functions == null || functions.size() != 1) {
- // topN only allow one measure
- return false;
+ if (!unmatchedDimensions.isEmpty()) {
+ logger.info("Exclude cube " + cube.getName() + " because unmatched dimensions");
+ return result;
}
-
- return isMatchedWithTopN(dimensionColumns, cube, digest);
- }
-
- /**
- * This is the method for internal, consumer need do the check as the public method did
- * @param dimensionColumns
- * @param cube
- * @param digest
- * @return
- */
- private static boolean isMatchedWithTopN(Collection<TblColRef> dimensionColumns, CubeInstance cube, SQLDigest digest) {
-
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> functions = digest.aggregations;
- FunctionDesc onlyFunction = functions.iterator().next();
- if (onlyFunction.isSum() == false) {
- // topN only support SUM expression
- return false;
+ if (!unmatchedAggregations.isEmpty()) {
+ logger.info("Exclude cube " + cube.getName() + " because unmatched aggregations");
+ return result;
}
- Collection<TblColRef> dimensionColumnsCopy = new ArrayList<TblColRef>(dimensionColumns);
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- if (measure.getFunction().isTopN()) {
- List<TblColRef> cols = measure.getFunction().getParameter().getColRefs();
- TblColRef literalCol = cols.get(cols.size() - 1);
- if (digest.groupbyColumns.contains(literalCol)) {
- dimensionColumnsCopy.remove(literalCol);
- if (isMatchedWithDimensions(dimensionColumnsCopy, cube)) {
- if (measure.getFunction().isTopNCompatibleSum(onlyFunction)) {
- return true;
- }
- }
- dimensionColumnsCopy.add(literalCol);
- }
- }
- }
-
- return false;
+ // cost will be minded by caller
+ result.capable = true;
+ return result;
}
- private static boolean hasTopNMeasure(CubeDesc cubeDesc) {
- for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
- if (measureDesc.getFunction().isTopN())
- return true;
- }
+ private static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) {
+ Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns;
+ Collection<TblColRef> filterColumns = sqlDigest.filterColumns;
- return false;
+ Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
+ dimensionColumns.addAll(groupByColumns);
+ dimensionColumns.addAll(filterColumns);
+ return dimensionColumns;
}
- private static boolean isMatchedWithDimensions(Collection<TblColRef> dimensionColumns, CubeInstance cube) {
+ private static Set<TblColRef> unmatchedDimensions(Collection<TblColRef> dimensionColumns, CubeInstance cube) {
+ HashSet<TblColRef> result = Sets.newHashSet(dimensionColumns);
CubeDesc cubeDesc = cube.getDescriptor();
- boolean matchAgg = cubeDesc.listDimensionColumnsIncludingDerived().containsAll(dimensionColumns);
- return matchAgg;
+ result.removeAll(cubeDesc.listDimensionColumnsIncludingDerived());
+ return result;
}
- private static boolean isMatchedWithAggregations(Collection<FunctionDesc> aggregations, CubeInstance cube) {
+ private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, CubeInstance cube) {
+ HashSet<FunctionDesc> result = Sets.newHashSet(aggregations);
CubeDesc cubeDesc = cube.getDescriptor();
- boolean matchAgg = cubeDesc.listAllFunctions().containsAll(aggregations);
- return matchAgg;
+ result.removeAll(cubeDesc.listAllFunctions());
+ return result;
}
- private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) {
+ private static boolean isJoinMatch(Collection<JoinDesc> joins, CubeInstance cube) {
CubeDesc cubeDesc = cube.getDescriptor();
List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
@@ -204,30 +150,55 @@ public class CubeCapabilityChecker {
return true;
}
- private static boolean isWeaklyMatchedWithAggregations(Collection<FunctionDesc> aggregations, Collection<TblColRef> metricColumns, CubeInstance cube) {
+ private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
CubeDesc cubeDesc = cube.getDescriptor();
Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
- boolean matched = true;
- for (FunctionDesc functionDesc : aggregations) {
- if (cubeFuncs.contains(functionDesc))
+ Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
+ while (it.hasNext()) {
+ FunctionDesc functionDesc = it.next();
+
+ if (cubeFuncs.contains(functionDesc)) {
+ it.remove();
continue;
+ }
// only inverted-index cube does not have count, and let calcite handle in this case
- if (functionDesc.isCount())
+ if (functionDesc.isCount()) {
+ it.remove();
continue;
+ }
- if (functionDesc.isCountDistinct()) // calcite can not handle distinct count
- matched = false;
-
- TblColRef col = null;
- if (functionDesc.getParameter().getColRefs().size() > 0)
- col = functionDesc.getParameter().getColRefs().get(0);
+ // calcite can not handle distinct count
+ if (functionDesc.isCountDistinct()) {
+ continue;
+ }
- if (col == null || !cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
- matched = false;
+ // calcite can do aggregation from columns on-the-fly
+ List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
+ if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) {
+ result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
+ it.remove();
+ continue;
}
}
- return matched;
}
+
+ // custom measure types can cover unmatched dimensions or measures
+ private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
+ CubeDesc cubeDesc = cube.getDescriptor();
+ for (MeasureDesc measure : cubeDesc.getMeasures()) {
+ if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
+ break;
+
+ MeasureType measureType = MeasureType.create(measure.getFunction());
+ if (measureType instanceof BasicMeasureType)
+ continue;
+
+ CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure);
+ if (inf != null)
+ result.influences.add(inf);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
deleted file mode 100644
index 138d01e..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import java.util.Collection;
-import java.util.HashSet;
-
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-/**
- *
- * the unified logic for defining a sql's dimension
- */
-public class CubeDimensionDeriver {
-
- public static Collection<TblColRef> getDimensionColumns(SQLDigest sqlDigest) {
- Collection<TblColRef> groupByColumns = sqlDigest.groupbyColumns;
- Collection<TblColRef> filterColumns = sqlDigest.filterColumns;
-
- Collection<MeasureDesc> sortMeasures = sqlDigest.sortMeasures;
- Collection<SQLDigest.OrderEnum> sortOrders = sqlDigest.sortOrders;
-
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2208136..81c7909 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -32,6 +32,8 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -339,12 +341,20 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
- public boolean isCapable(SQLDigest digest) {
- return CubeCapabilityChecker.check(this, digest, true);
+ public CapabilityResult isCapable(SQLDigest digest) {
+ CapabilityResult result = CubeCapabilityChecker.check(this, digest);
+ if (result.capable) {
+ result.cost = getCost(digest);
+ for (CapabilityInfluence i : result.influences) {
+ result.cost *= (i.suggestCostMultiplier() == 0) ? 1.0 : i.suggestCostMultiplier();
+ }
+ } else {
+ result.cost = -1;
+ }
+ return result;
}
- @Override
- public int getCost(SQLDigest digest) {
+ private int getCost(SQLDigest digest) {
int calculatedCost = cost;
calculatedCost += getAllDimensions().size() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
@@ -357,11 +367,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
}
- if (CubeCapabilityChecker.isMatchedWithTopN(this, digest)) {
- // this is topN query
- calculatedCost = calculatedCost / 3;
- }
-
return calculatedCost;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 4fe59c0..8e7de6f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -18,6 +18,7 @@
package org.apache.kylin.measure;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -29,6 +30,8 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
import com.google.common.collect.Maps;
@@ -83,12 +86,18 @@ abstract public class MeasureType {
abstract public MeasureAggregator<?> newAggregator();
- abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
-
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ return null;
+ }
+
/* ============================================================================
* Cube Selection
* ---------------------------------------------------------------------------- */
+ public CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) {
+ return null;
+ }
+
/* ============================================================================
* Query
* ---------------------------------------------------------------------------- */
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index fe53bab..f314870 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -18,8 +18,6 @@
package org.apache.kylin.measure.basic;
-import java.util.List;
-
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
@@ -30,7 +28,6 @@ import org.apache.kylin.metadata.datatype.DoubleSerializer;
import org.apache.kylin.metadata.datatype.LongSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
public class BasicMeasureType extends MeasureType {
@@ -119,10 +116,4 @@ public class BasicMeasureType extends MeasureType {
return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
}
- @Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 2ad7630..4a73478 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -18,7 +18,6 @@
package org.apache.kylin.measure.hllc;
-import java.util.List;
import java.util.Map;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
@@ -90,10 +89,4 @@ public class HLLCMeasureType extends MeasureType {
return new LDCAggregator();
}
- @Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 561f9f1..d6c5a6f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,6 +18,7 @@
package org.apache.kylin.measure.topn;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -33,8 +34,11 @@ import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.hllc.HLLCSerializer;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
public class TopNMeasureType extends MeasureType {
@@ -43,9 +47,9 @@ public class TopNMeasureType extends MeasureType {
public TopNMeasureType(DataType dataType) {
if ("topn".equals(dataType.getName()) == false)
throw new IllegalArgumentException();
-
+
this.dataType = dataType;
-
+
if (this.dataType.getPrecision() < 1 || this.dataType.getPrecision() > 1000)
throw new IllegalArgumentException("TopN precision must be between 1 and 1000");
}
@@ -59,11 +63,11 @@ public class TopNMeasureType extends MeasureType {
public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
return HLLCSerializer.class;
}
-
+
@Override
public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
// TODO Auto-generated method stub
-
+
}
@SuppressWarnings("rawtypes")
@@ -74,10 +78,10 @@ public class TopNMeasureType extends MeasureType {
public TopNCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length != 2)
throw new IllegalArgumentException();
-
+
double counter = values[0] == null ? 0 : Double.parseDouble(values[0]);
String literal = values[1];
-
+
// encode literal using dictionary
TblColRef literalCol = measureDesc.getFunction().getTopNLiteralColumn();
Dictionary<String> dictionary = dictionaryMap.get(literalCol);
@@ -90,7 +94,7 @@ public class TopNMeasureType extends MeasureType {
topNCounter.offer(key, counter);
return topNCounter;
}
-
+
@SuppressWarnings("unchecked")
@Override
public TopNCounter reEncodeDictionary(TopNCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
@@ -135,4 +139,32 @@ public class TopNMeasureType extends MeasureType {
return Collections.singletonList(literalCol);
}
+ @Override
+ public CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc topN) {
+ // TopN measure can (and only can) provide one numeric measure and one literal dimension
+ // e.g. select seller, sum(gmv) from ... group by seller order by 2 desc limit 100
+
+ // check digest requires only one measure
+ if (digest.aggregations.size() != 1)
+ return null;
+
+ // the measure function must be SUM
+ FunctionDesc onlyFunction = digest.aggregations.iterator().next();
+ if (onlyFunction.isSum() == false)
+ return null;
+
+ TblColRef literalCol = topN.getFunction().getTopNLiteralColumn();
+ if (unmatchedDimensions.contains(literalCol) && topN.getFunction().isTopNCompatibleSum(onlyFunction)) {
+ unmatchedDimensions.remove(literalCol);
+ unmatchedAggregations.remove(onlyFunction);
+ return new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 0.3; // make sure TopN get ahead of other matched realizations
+ }
+ };
+ } else
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
new file mode 100644
index 0000000..cc5bae7
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/CapabilityResult.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.realization;
+
+import java.util.List;
+
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+import com.google.common.collect.Lists;
+
+public class CapabilityResult {
+
+ /** Is capable or not */
+ public boolean capable;
+
+ /** The smaller the cost, the more capable the realization */
+ public int cost;
+
+ /**
+ * For info purpose, marker objects to indicate all special features
+ * (dimension-as-measure, topN etc.) that have decided the capability.
+ */
+ public List<CapabilityInfluence> influences = Lists.newArrayListWithCapacity(1);
+
+ public static interface CapabilityInfluence {
+ /** Suggest a multiplier to influence query cost */
+ double suggestCostMultiplier();
+ }
+
+ public static class DimensionAsMeasure implements CapabilityInfluence {
+
+ final FunctionDesc function;
+
+ public DimensionAsMeasure(FunctionDesc function) {
+ this.function = function;
+ }
+
+ @Override
+ public double suggestCostMultiplier() {
+ return 1.0;
+ }
+
+ public FunctionDesc getMeasureFunction() {
+ return function;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index ccfa956..1857c85 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -26,26 +26,14 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
public interface IRealization extends IStorageAware {
-
- public boolean isCapable(SQLDigest digest);
-
+
/**
- * Given the features of a query, return an integer indicating how capable the realization
- * is to answer the query.
- *
- * @return -1 if the realization cannot fulfill the query;
- * or a number between 0-100 if the realization can answer the query, the smaller
- * the number, the more efficient the realization.
- * Especially,
- * 0 - means the realization has the exact result pre-calculated, no less no more;
- * 100 - means the realization will scan the full table with little or no indexing.
+ * Given the features of a query, check how capable the realization is to answer the query.
*/
- public int getCost(SQLDigest digest);
+ public CapabilityResult isCapable(SQLDigest digest);
/**
* Get whether this specific realization is a cube or InvertedIndex
- *
- * @return
*/
public RealizationType getType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 2a6e5d3..98330ec 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -29,6 +29,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -152,24 +153,23 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
- public boolean isCapable(SQLDigest digest) {
+ public CapabilityResult isCapable(SQLDigest digest) {
+ CapabilityResult result = new CapabilityResult();
+ result.cost = Integer.MAX_VALUE;
+
for (IRealization realization : getRealizations()) {
- if (realization.isCapable(digest))
- return true;
- }
- return false;
- }
-
- @Override
- public int getCost(SQLDigest digest) {
- cost = Integer.MAX_VALUE;
- for (IRealization realization : this.getRealizations()) {
- if (realization.isCapable(digest))
- cost = Math.min(cost, realization.getCost(digest));
+ CapabilityResult child = realization.isCapable(digest);
+ if (child.capable) {
+ result.capable = true;
+ result.cost = Math.min(result.cost, child.cost);
+ result.influences.addAll(child.influences);
+ }
}
-
- // Make hybrid always win its children
- return cost - 1;
+
+ if (result.cost > 0)
+ result.cost--; // let hybrid win its children
+
+ return result;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
index 49f9fd9..8911c0e 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
@@ -49,7 +49,7 @@ public class HybridStorageQuery implements IStorageQuery {
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
List<ITupleIterator> tupleIterators = Lists.newArrayList();
for (int i = 0; i < realizations.length; i++) {
- if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest)) {
+ if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest).capable) {
ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest, returnTupleInfo);
tupleIterators.add(dataIterator);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index 0ba07ee..114904a 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -26,7 +26,14 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IBuildable;
+import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -270,15 +277,21 @@ public class IIInstance extends RootPersistentEntity implements IRealization, IB
}
@Override
- public boolean isCapable(SQLDigest digest) {
- if (!digest.factTable.equalsIgnoreCase(this.getFactTable()))
- return false;
-
- return IICapabilityChecker.check(this, digest);
+ public CapabilityResult isCapable(SQLDigest digest) {
+ CapabilityResult result = new CapabilityResult();
+
+ if (!digest.factTable.equalsIgnoreCase(this.getFactTable())) {
+ result.capable = false;
+ } else {
+ result.capable = IICapabilityChecker.check(this, digest);
+ if (result.capable)
+ result.cost = getCost(digest);
+ }
+
+ return result;
}
- @Override
- public int getCost(SQLDigest digest) {
+ private int getCost(SQLDigest digest) {
int calculatedCost = cost;
for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
new file mode 100644
index 0000000..5e4dcf6
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing;
+
+import java.util.Map;
+
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+
+import com.google.common.collect.Maps;
+
+public class Candidate implements Comparable<Candidate> {
+
+ static final Map<RealizationType, Integer> PRIORITIES = Maps.newHashMap();
+
+ static {
+ PRIORITIES.put(RealizationType.HYBRID, 0);
+ PRIORITIES.put(RealizationType.CUBE, 0);
+ PRIORITIES.put(RealizationType.INVERTED_INDEX, 1);
+ }
+
+ /** for test only */
+ public static void setPriorities(Map<RealizationType, Integer> priorities) {
+ PRIORITIES.clear();
+ PRIORITIES.putAll(priorities);
+ }
+
+ // ============================================================================
+
+ IRealization realization;
+ SQLDigest sqlDigest;
+ int priority;
+ CapabilityResult capability;
+
+ public Candidate(IRealization realization, SQLDigest sqlDigest) {
+ this.realization = realization;
+ this.sqlDigest = sqlDigest;
+ this.priority = PRIORITIES.get(realization.getType());
+ }
+
+ public IRealization getRealization() {
+ return realization;
+ }
+
+ public SQLDigest getSqlDigest() {
+ return sqlDigest;
+ }
+
+ public int getPriority() {
+ return priority;
+ }
+
+ public CapabilityResult getCapability() {
+ return capability;
+ }
+
+ public void setCapability(CapabilityResult capability) {
+ this.capability = capability;
+ }
+
+ @Override
+ public int compareTo(Candidate o) {
+ int comp = this.priority - o.priority;
+ if (comp != 0) {
+ return comp;
+ }
+
+ comp = this.capability.cost - o.capability.cost;
+ if (comp != 0) {
+ return comp;
+ }
+
+ if (this.realization instanceof HybridInstance)
+ return -1;
+ else if (o.realization instanceof HybridInstance)
+ return 1;
+ else
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index 7f2bb49..7493e08 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -19,10 +19,17 @@
package org.apache.kylin.query.routing;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.CapabilityResult.DimensionAsMeasure;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.relnode.OLAPContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,20 +49,51 @@ public class QueryRouter {
logger.info("The project manager's reference is " + prjMgr);
String factTableName = olapContext.firstTableScan.getTableName();
String projectName = olapContext.olapSchema.getProjectName();
- List<IRealization> realizations = Lists.newArrayList(prjMgr.getRealizationsByTable(projectName, factTableName));
- logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(realizations, ","));
+ Set<IRealization> realizations = prjMgr.getRealizationsByTable(projectName, factTableName);
+ SQLDigest sqlDigest = olapContext.getSQLDigest();
- //rule based realization selection, rules might reorder realizations or remove specific realization
- RoutingRule.applyRules(realizations, olapContext);
+ List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
+ for (IRealization real : realizations) {
+ if (real.isReady())
+ candidates.add(new Candidate(real, sqlDigest));
+ }
+
+ logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(candidates, ","));
- if (realizations.size() == 0) {
- throw new NoRealizationFoundException("Can't find any realization. Please confirm with providers. SQL digest: " + olapContext.getSQLDigest().toString());
+ // rule based realization selection, rules might reorder realizations or remove specific realization
+ RoutingRule.applyRules(candidates);
+
+ if (candidates.size() == 0) {
+ throw new NoRealizationFoundException("Can't find any realization. Please confirm with providers. SQL digest: " + sqlDigest.toString());
}
+ Candidate chosen = candidates.get(0);
+ adjustForDimensionAsMeasure(chosen, olapContext);
+
logger.info("The realizations remaining: ");
- logger.info(RoutingRule.getPrintableText(realizations));
- logger.info("The realization being chosen: " + realizations.get(0).getName());
+ logger.info(RoutingRule.getPrintableText(candidates));
+ logger.info("The realization being chosen: " + chosen.realization.getName());
- return realizations.get(0);
+ return chosen.realization;
}
+
+ private static void adjustForDimensionAsMeasure(Candidate chosen, OLAPContext olapContext) {
+ CapabilityResult capability = chosen.getCapability();
+ for (CapabilityInfluence inf : capability.influences) {
+ // convert the metric to dimension
+ if (inf instanceof DimensionAsMeasure) {
+ FunctionDesc functionDesc = ((DimensionAsMeasure) inf).getMeasureFunction();
+ functionDesc.setDimensionAsMetric(true);
+ olapContext.rewriteFields.remove(functionDesc.getRewriteFieldName());
+ for (TblColRef col : functionDesc.getParameter().getColRefs()) {
+ if (col != null) {
+ olapContext.metricsColumns.remove(col);
+ olapContext.groupByColumns.add(col);
+ }
+ }
+ logger.info("Adjust DimensionAsMeasure for " + functionDesc);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index 1cd55d4..715f6d1 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -23,10 +23,8 @@ import java.util.List;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.routing.RoutingRules.AdjustForWeaklyMatchedRealization;
-import org.apache.kylin.query.routing.RoutingRules.RealizationSortRule;
-import org.apache.kylin.query.routing.RoutingRules.RemoveUncapableRealizationsRule;
+import org.apache.kylin.query.routing.rules.RealizationSortRule;
+import org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,23 +42,23 @@ public abstract class RoutingRule {
static {
rules.add(new RemoveUncapableRealizationsRule());
rules.add(new RealizationSortRule());
- rules.add(new AdjustForWeaklyMatchedRealization());//this rule might modify olapcontext content, better put it at last
}
- public static void applyRules(List<IRealization> realizations, OLAPContext olapContext) {
+ public static void applyRules(List<Candidate> candidates) {
for (RoutingRule rule : rules) {
- logger.info("Realizations order before: " + getPrintableText(realizations));
+ logger.info("Realizations order before: " + getPrintableText(candidates));
logger.info("Applying rule : " + rule);
- rule.apply(realizations, olapContext);
- logger.info("Realizations order after: " + getPrintableText(realizations));
+ rule.apply(candidates);
+ logger.info("Realizations order after: " + getPrintableText(candidates));
logger.info("===================================================");
}
}
- public static String getPrintableText(List<IRealization> realizations) {
+ public static String getPrintableText(List<Candidate> candidates) {
StringBuffer sb = new StringBuffer();
sb.append("[");
- for (IRealization r : realizations) {
+ for (Candidate candidate : candidates) {
+ IRealization r = candidate.realization;
sb.append(r.getName());
sb.append(",");
}
@@ -108,6 +106,6 @@ public abstract class RoutingRule {
return this.getClass().toString();
}
- public abstract void apply(List<IRealization> realizations, OLAPContext olapContext);
+ public abstract void apply(List<Candidate> candidates);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
deleted file mode 100644
index 250f2a6..0000000
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.routing.RoutingRules;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.kylin.cube.CubeCapabilityChecker;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.routing.RoutingRule;
-import org.apache.kylin.storage.hybrid.HybridInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- */
-public class AdjustForWeaklyMatchedRealization extends RoutingRule {
- private static final Logger logger = LoggerFactory.getLogger(AdjustForWeaklyMatchedRealization.class);
-
- @Override
- public void apply(List<IRealization> realizations, OLAPContext olapContext) {
- if (realizations.size() > 0) {
- IRealization first = realizations.get(0);
-
- if (first instanceof HybridInstance) {
- HybridInstance hybrid = (HybridInstance) first;
- first = hybrid.getLatestRealization();
- }
-
- if (first instanceof CubeInstance) {
- CubeInstance cube = (CubeInstance) first;
- adjustOLAPContextIfNecessary(cube, olapContext);
- }
-
- if (first instanceof IIInstance) {
- IIInstance ii = (IIInstance) first;
- adjustOLAPContextIfNecessary(ii, olapContext);
- }
- }
- }
-
- private static void adjustOLAPContextIfNecessary(IIInstance ii, OLAPContext olapContext) {
- IIDesc iiDesc = ii.getDescriptor();
- Collection<FunctionDesc> iiFuncs = iiDesc.listAllFunctions();
- convertAggregationToDimension(olapContext, iiFuncs);
- }
-
- private static void adjustOLAPContextIfNecessary(CubeInstance cube, OLAPContext olapContext) {
- if (CubeCapabilityChecker.check(cube, olapContext.getSQLDigest(), false))
- return;
-
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
- convertAggregationToDimension(olapContext, cubeFuncs);
- }
-
- private static void convertAggregationToDimension(OLAPContext olapContext, Collection<FunctionDesc> availableAggregations) {
- Iterator<FunctionDesc> it = olapContext.aggregations.iterator();
- while (it.hasNext()) {
- FunctionDesc functionDesc = it.next();
- if (!availableAggregations.contains(functionDesc)) {
- // try to convert the metric to dimension to see if it works
- TblColRef col = functionDesc.getParameter().getColRefs().get(0);
- functionDesc.setDimensionAsMetric(true);
- olapContext.rewriteFields.remove(functionDesc.getRewriteFieldName());
- if (col != null) {
- olapContext.metricsColumns.remove(col);
- olapContext.groupByColumns.add(col);
- }
- logger.info("Adjust OLAPContext for " + functionDesc);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
deleted file mode 100644
index 6e196d9..0000000
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.routing.RoutingRules;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.routing.RoutingRule;
-
-import com.google.common.collect.Maps;
-
-/**
- */
-public class RealizationPriorityRule extends RoutingRule {
- static Map<RealizationType, Integer> priorities = Maps.newHashMap();
-
- static {
- priorities.put(RealizationType.HYBRID, 0);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- }
-
- public static void setPriorities(Map<RealizationType, Integer> priorities) {
- RealizationPriorityRule.priorities = priorities;
- }
-
- public void apply(List<IRealization> realizations, OLAPContext olapContext) {
-
- Collections.sort(realizations, new Comparator<IRealization>() {
- @Override
- public int compare(IRealization o1, IRealization o2) {
- int i1 = priorities.get(o1.getType());
- int i2 = priorities.get(o2.getType());
- return i1 - i2;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
deleted file mode 100644
index d101288..0000000
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationSortRule.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.routing.RoutingRules;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.routing.RoutingRule;
-import org.apache.kylin.storage.hybrid.HybridInstance;
-
-/**
- */
-public class RealizationSortRule extends RoutingRule {
- @Override
- public void apply(List<IRealization> realizations, final OLAPContext olapContext) {
-
- // sort cube candidates, 0) the priority 1) the cost indicator, 2) the lesser header
- // columns the better, 3) the lesser body columns the better 4) the larger date range the better
-
- Collections.sort(realizations, new Comparator<IRealization>() {
- @Override
- public int compare(IRealization o1, IRealization o2) {
- int i1 = RealizationPriorityRule.priorities.get(o1.getType());
- int i2 = RealizationPriorityRule.priorities.get(o2.getType());
- int comp = i1 - i2;
- if (comp != 0) {
- return comp;
- }
-
- comp = o1.getCost(olapContext.getSQLDigest()) - o2.getCost(olapContext.getSQLDigest());
- if (comp != 0) {
- return comp;
- }
-
- if (o1 instanceof HybridInstance)
- return -1;
- else if (o2 instanceof HybridInstance)
- return 1;
-
- return 0;
- }
- });
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
deleted file mode 100644
index c710ff1..0000000
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RemoveUncapableRealizationsRule.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.routing.RoutingRules;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.routing.RoutingRule;
-
-/**
- */
-public class RemoveUncapableRealizationsRule extends RoutingRule {
- @Override
- public void apply(List<IRealization> realizations, OLAPContext olapContext) {
- for (Iterator<IRealization> iterator = realizations.iterator(); iterator.hasNext();) {
- IRealization realization = iterator.next();
- if (!realization.isCapable(olapContext.getSQLDigest())) {
- iterator.remove();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/rules/RealizationSortRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RealizationSortRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RealizationSortRule.java
new file mode 100644
index 0000000..d3c67d7
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RealizationSortRule.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing.rules;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.RoutingRule;
+
+/**
+ */
+public class RealizationSortRule extends RoutingRule {
+ @Override
+ public void apply(List<Candidate> candidates) {
+ Collections.sort(candidates);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
new file mode 100644
index 0000000..576b47f
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveUncapableRealizationsRule.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.routing.rules;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.kylin.metadata.realization.CapabilityResult;
+import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.RoutingRule;
+
+/**
+ */
+public class RemoveUncapableRealizationsRule extends RoutingRule {
+ @Override
+ public void apply(List<Candidate> candidates) {
+ for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) {
+ Candidate candidate = iterator.next();
+
+ CapabilityResult capability = candidate.getRealization().isCapable(candidate.getSqlDigest());
+ if (capability.capable)
+ candidate.setCapability(capability);
+ else
+ iterator.remove();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4c4c88a/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
index 78b3d1c..344433a 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
@@ -23,10 +23,9 @@ import java.util.Collection;
import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
+import org.apache.kylin.query.routing.Candidate;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -45,7 +44,7 @@ public class ITIIQueryTest extends ITKylinQueryTest {
priorities.put(RealizationType.INVERTED_INDEX, 0);
priorities.put(RealizationType.CUBE, 1);
priorities.put(RealizationType.HYBRID, 1);
- RealizationPriorityRule.setPriorities(priorities);
+ Candidate.setPriorities(priorities);
}
@@ -57,7 +56,7 @@ public class ITIIQueryTest extends ITKylinQueryTest {
priorities.put(RealizationType.INVERTED_INDEX, 1);
priorities.put(RealizationType.CUBE, 0);
priorities.put(RealizationType.HYBRID, 0);
- RealizationPriorityRule.setPriorities(priorities);
+ Candidate.setPriorities(priorities);
}
@Parameterized.Parameters
[18/21] kylin git commit: KYLIN-976 Clean up,
remove all explict check on TopN & HLLC
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index fbb258f..42946eb 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -166,7 +166,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
List<Map<TblColRef, String>> fuzzyValues = FuzzyValueCombination.calculate(fuzzyValueSet, FUZZY_VALUE_CAP);
for (Map<TblColRef, String> fuzzyValue : fuzzyValues) {
- result.add(new Pair<byte[], byte[]>(fuzzyKeyEncoder.encode(fuzzyValue), fuzzyMaskEncoder.encode(fuzzyValue)));
+ result.add(Pair.newPair(fuzzyKeyEncoder.encode(fuzzyValue), fuzzyMaskEncoder.encode(fuzzyValue)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index 152b4af..f2b34fc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -94,6 +94,6 @@ public class DFSFileTable implements ReadableTable {
lastModified = Math.max(lastModified, file.getModificationTime());
}
- return new Pair<Long, Long>(size, lastModified);
+ return Pair.newPair(size, lastModified);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 45cc88e..3dddece 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -157,8 +157,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
int colParamIdx = 0; // index among parameters of column type
for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
String value;
- if (function.isCount() || function.isHolisticCountDistinct()) {
- // note for holistic count distinct, this value will be ignored
+ if (function.isCount()) {
value = "1";
} else if (param.isColumnType()) {
value = getCell(colIdxOnFlatTable[colParamIdx++], splitBuffers);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index 3edaefb..daa610f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -170,9 +170,8 @@ public class CuboidJob extends AbstractHadoopJob {
// number of reduce tasks
int numReduceTasks = (int) Math.round(totalReduceInputMB / perReduceInputMB * reduceCountRatio);
- // adjust reducer number for cube which has DISTINCT_COUNT measures for
- // better performance
- if (cubeDesc.hasHolisticCountDistinctMeasures()) {
+ // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
+ if (cubeDesc.hasMemoryHungryMeasures()) {
numReduceTasks = numReduceTasks * 4;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index b1527e6..18bce34 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -124,7 +124,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
- dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
+ dictMeasures.add(Pair.newPair(i, measureType.newIngester()));
}
}
if (dictMeasures.size() > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index cf5049a..682a91c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -126,7 +126,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
- dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
+ dictMeasures.add(Pair.newPair(i, measureType.newIngester()));
}
}
if (dictMeasures.size() > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 8f65691..c27b3f1 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -63,6 +63,7 @@ import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.measure.MeasureAggregators;
import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -324,11 +325,11 @@ public class SparkCubing extends AbstractApplication {
}
for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
- if (measureDesc.getFunction().isTopN()) {
- List<TblColRef> colRefs = measureDesc.getFunction().getParameter().getColRefs();
- TblColRef col = colRefs.get(colRefs.size() - 1);
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
+ FunctionDesc func = measureDesc.getFunction();
+ List<TblColRef> colRefs = func.getMeasureType().getColumnsNeedDictionary(func);
+ for (TblColRef col : colRefs) {
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
index c9a78a4..bd952a1 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
@@ -20,12 +20,13 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
+import org.apache.kylin.measure.hllc.HLLCMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
abstract public class FixedLenMeasureCodec<T> {
public static FixedLenMeasureCodec<?> get(DataType type) {
- if (type.isHLLC()) {
+ if (HLLCMeasureType.DATATYPE_HLLC.equals(type.getName())) {
return new FixedHLLCodec(type);
} else {
return new FixedPointLongCodec(type);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
deleted file mode 100644
index 16748a6..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.Tuple;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.translate.HBaseKeyRange;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- */
-public class CubeSegmentTopNTupleIterator extends CubeSegmentTupleIterator{
-
- private Iterator<Tuple> innerResultIterator;
-
- public CubeSegmentTopNTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
- Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, TblColRef topNCol, //
- List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
- super(cubeSeg, keyRanges, conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
- this.tupleConverter = new CubeTupleConverter(cubeSeg, cuboid, rowValueDecoders, returnTupleInfo, topNCol);
- }
-
- @Override
- public boolean hasNext() {
- if (next != null)
- return true;
-
-
- if (innerResultIterator == null) {
- if (resultIterator == null) {
- if (rangeIterator.hasNext() == false)
- return false;
-
- resultIterator = doScan(rangeIterator.next());
- }
-
- if (resultIterator.hasNext() == false) {
- closeScanner();
- resultIterator = null;
- innerResultIterator = null;
- return hasNext();
- }
-
- Result result = resultIterator.next();
- scanCount++;
- if (++scanCountDelta >= 1000)
- flushScanCountDelta();
- innerResultIterator = tupleConverter.translateTopNResult(result, oneTuple);
- }
-
- if (innerResultIterator.hasNext()) {
- next = innerResultIterator.next();
- return true;
- } else {
- innerResultIterator = null;
- return hasNext();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 4acd8f8..50cfefa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -99,7 +99,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
assert cuboid.equals(range.getCuboid());
}
- this.tupleConverter = new CubeTupleConverter(cubeSeg, cuboid, rowValueDecoders, returnTupleInfo, null);
+ this.tupleConverter = new CubeTupleConverter(cubeSeg, cuboid, rowValueDecoders, returnTupleInfo);
this.oneTuple = new Tuple(returnTupleInfo);
this.rangeIterator = keyRanges.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 063e254..c115f9c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -48,6 +48,7 @@ import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
@@ -95,9 +96,8 @@ public class CubeStorageQuery implements ICachableStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-
- // check whether this is a TopN query
- checkAndRewriteTopN(sqlDigest);
+ // allow custom measures hack
+ notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
TupleFilter filter = sqlDigest.filter;
@@ -378,12 +378,8 @@ public class CubeStorageQuery implements ICachableStorageQuery {
for (HBaseColumnDesc hbCol : hbCols) {
bestHBCol = hbCol;
bestIndex = hbCol.findMeasure(aggrFunc);
- MeasureDesc measure = hbCol.getMeasures()[bestIndex];
- // criteria for holistic measure: Exact Aggregation && Exact Cuboid
- if (measure.getFunction().isHolisticCountDistinct() && context.isExactAggregation()) {
- logger.info("Holistic count distinct chosen for " + aggrFunc);
- break;
- }
+ // we used to prefer specific measure over another (holistic distinct count), now it's gone
+ break;
}
RowValueDecoder codec = codecMap.get(bestHBCol);
@@ -716,7 +712,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
private void setThreshold(Collection<TblColRef> dimensions, List<RowValueDecoder> valueDecoders, StorageContext context) {
- if (RowValueDecoder.hasMemHungryCountDistinct(valueDecoders) == false) {
+ if (RowValueDecoder.hasMemHungryMeasures(valueDecoders) == false) {
return;
}
@@ -754,33 +750,11 @@ public class CubeStorageQuery implements ICachableStorageQuery {
ObserverEnabler.enableCoprocessorIfBeneficial(cubeInstance, groupsCopD, valueDecoders, context);
}
- private void checkAndRewriteTopN(SQLDigest sqlDigest) {
- FunctionDesc topnFunc = null;
- TblColRef topnLiteralCol = null;
+ private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc func = measure.getFunction();
- if (func.isTopN() && sqlDigest.groupbyColumns.contains(func.getTopNLiteralColumn())) {
- topnFunc = func;
- topnLiteralCol = func.getTopNLiteralColumn();
- }
+ MeasureType<?> measureType = measure.getFunction().getMeasureType();
+ measureType.beforeStorageQuery(measure, sqlDigest);
}
-
- // if TopN is not involved
- if (topnFunc == null)
- return;
-
- if (sqlDigest.aggregations.size() != 1) {
- throw new IllegalStateException("When query with topN, only one metrics is allowed.");
- }
-
- FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
- if (origFunc.isSum() == false) {
- throw new IllegalStateException("When query with topN, only SUM function is allowed.");
- }
-
- sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.remove(topnLiteralCol);
- sqlDigest.metricColumns.add(topnLiteralCol);
- logger.info("Rewrite function " + origFunc + " to " + topnFunc);
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index a669e1b..e414e08 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -2,18 +2,12 @@ package org.apache.kylin.storage.hbase.cube.v1;
import java.io.IOException;
import java.util.BitSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -40,19 +34,14 @@ public class CubeTupleConverter {
final int[] dimensionTupleIdx;
final int[][] metricsMeasureIdx;
final int[][] metricsTupleIdx;
- final TblColRef topNCol;
- int topNColTupleIdx;
- int topNMeasureTupleIdx;
- Dictionary<String> topNColDict;
- public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, List<RowValueDecoder> rowValueDecoders, TupleInfo tupleInfo, TblColRef topNCol) {
+ public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, List<RowValueDecoder> rowValueDecoders, TupleInfo tupleInfo) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
this.tupleInfo = tupleInfo;
this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg);
this.rowValueDecoders = rowValueDecoders;
this.derivedColFillers = Lists.newArrayList();
- this.topNCol = topNCol;
List<TblColRef> dimCols = cuboid.getColumns();
@@ -92,13 +81,6 @@ public class CubeTupleConverter {
}
}
- if (this.topNCol != null) {
- this.topNColTupleIdx = tupleInfo.hasColumn(this.topNCol) ? tupleInfo.getColumnIndex(this.topNCol) : -1;
- this.topNMeasureTupleIdx = metricsTupleIdx[0][0];
-
- this.topNColDict = (Dictionary<String>)cubeSeg.getDictionary(this.topNCol);
- }
-
// prepare derived columns and filler
Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
@@ -112,46 +94,6 @@ public class CubeTupleConverter {
}
}
- public Iterator<Tuple> translateTopNResult(Result hbaseRow, Tuple tuple) {
- translateResult(hbaseRow, tuple);
- Object topNCounterObj = tuple.getAllValues()[topNMeasureTupleIdx];
- assert (topNCounterObj instanceof TopNCounter);
- return new TopNCounterTupleIterator(tuple, (TopNCounter) topNCounterObj);
- }
-
- private class TopNCounterTupleIterator implements Iterator {
-
- private Tuple tuple;
- private Iterator<Counter> topNCounterIterator;
- private Counter<ByteArray> counter;
-
- private TopNCounterTupleIterator(Tuple tuple, TopNCounter topNCounter) {
- this.tuple = tuple;
- this.topNCounterIterator = topNCounter.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return topNCounterIterator.hasNext();
- }
-
- @Override
- public Tuple next() {
- counter = topNCounterIterator.next();
- int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
- String colValue = topNColDict.getValueFromId(key);
- tuple.setDimensionValue(topNColTupleIdx, colValue);
- tuple.setMeasureValue(topNMeasureTupleIdx, counter.getCount());
-
- return tuple;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
public void translateResult(Result hbaseRow, Tuple tuple) {
try {
byte[] rowkey = hbaseRow.getRow();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index 3f92cb0..e8dd5b9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.client.HConnection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
@@ -70,7 +69,8 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
- this.segmentIteratorList.add(newCubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo));
+ CubeSegmentTupleIterator it = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
+ this.segmentIteratorList.add(it);
}
this.segmentIteratorIterator = this.segmentIteratorList.iterator();
@@ -81,16 +81,6 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
}
- private CubeSegmentTupleIterator newCubeSegmentTupleIterator(CubeSegment seg, List<HBaseKeyRange> keyRange, HConnection conn, Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, StorageContext context2, TupleInfo returnTupleInfo) {
- MeasureDesc topN = RowValueDecoder.findTopN(rowValueDecoders);
- if (topN != null) {
- TblColRef topNCol = topN.getFunction().getTopNLiteralColumn();
- return new CubeSegmentTopNTupleIterator(seg, keyRange, conn, dimensions, filter, groupBy, topNCol, rowValueDecoders, context, returnTupleInfo);
- } else {
- return new CubeSegmentTupleIterator(seg, keyRange, conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
- }
- }
-
private Map<CubeSegment, List<HBaseKeyRange>> makeRangesMap(List<HBaseKeyRange> segmentKeyRanges) {
Map<CubeSegment, List<HBaseKeyRange>> map = Maps.newHashMap();
for (HBaseKeyRange range : segmentKeyRanges) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
index 521f111..5b16b04 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
@@ -32,6 +32,9 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
@@ -122,6 +125,8 @@ public class ObserverAggregators {
final int nHCols;
final ByteBuffer[] hColValues;
final int nTotalMeasures;
+
+ MeasureType measureTypes[];
public ObserverAggregators(HCol[] _hcols) {
this.hcols = sort(_hcols);
@@ -150,11 +155,18 @@ public class ObserverAggregators {
}
public MeasureAggregator[] createBuffer() {
+ if (measureTypes == null) {
+ measureTypes = new MeasureType[nTotalMeasures];
+ int i = 0;
+ for (HCol col : hcols) {
+ for (int j = 0; j < col.nMeasures; j++)
+ measureTypes[i++] = MeasureTypeFactory.create(col.funcNames[j], DataType.getType(col.dataTypes[j]));
+ }
+ }
+
MeasureAggregator[] aggrs = new MeasureAggregator[nTotalMeasures];
- int i = 0;
- for (HCol col : hcols) {
- for (int j = 0; j < col.nMeasures; j++)
- aggrs[i++] = MeasureAggregator.create(col.funcNames[j], col.dataTypes[j]);
+ for (int i = 0; i < nTotalMeasures; i++) {
+ aggrs[i] = measureTypes[i].newAggregator();
}
return aggrs;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
index db05a40..ea33f9a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
@@ -80,7 +80,7 @@ public class FuzzyRowFilterV2 extends FilterBase {
for (int i = 0; i < fuzzyKeysData.size(); i++) {
p = fuzzyKeysData.get(i);
if (p.getFirst().length != p.getSecond().length) {
- Pair<String, String> readable = new Pair<String, String>(Bytes.toStringBinary(p.getFirst()), Bytes.toStringBinary(p.getSecond()));
+ Pair<String, String> readable = Pair.newPair(Bytes.toStringBinary(p.getFirst()), Bytes.toStringBinary(p.getSecond()));
throw new IllegalArgumentException("Fuzzy pair lengths do not match: " + readable);
}
// update mask ( 0 -> -1 (0xff), 1 -> 0)
@@ -225,7 +225,7 @@ public class FuzzyRowFilterV2 extends FilterBase {
void updateWith(Cell currentCell, Pair<byte[], byte[]> fuzzyData) {
byte[] nextRowKeyCandidate = getNextForFuzzyRule(isReversed(), currentCell.getRowArray(), currentCell.getRowOffset(), currentCell.getRowLength(), fuzzyData.getFirst(), fuzzyData.getSecond());
if (nextRowKeyCandidate != null) {
- nextRows.add(new Pair<byte[], Pair<byte[], byte[]>>(nextRowKeyCandidate, fuzzyData));
+ nextRows.add(Pair.newPair(nextRowKeyCandidate, fuzzyData));
}
}
@@ -263,7 +263,7 @@ public class FuzzyRowFilterV2 extends FilterBase {
FilterProtosExt.BytesBytesPair current = proto.getFuzzyKeysData(i);
byte[] keyBytes = current.getFirst().toByteArray();
byte[] keyMeta = current.getSecond().toByteArray();
- fuzzyKeysData.add(new Pair<byte[], byte[]>(keyBytes, keyMeta));
+ fuzzyKeysData.add(Pair.newPair(keyBytes, keyMeta));
}
return new FuzzyRowFilterV2(fuzzyKeysData);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 412e7602..6a6c887 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -134,7 +134,7 @@ public abstract class CubeHBaseRPC {
fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey);
fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask);
- ret.add(new Pair<byte[], byte[]>(hbaseFuzzyKey, hbaseFuzzyMask));
+ ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask));
}
return ret;
@@ -153,7 +153,7 @@ public abstract class CubeHBaseRPC {
for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
if (selectedColBlocks.get(colBlkIndex)) {
byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
- result.add(new Pair<byte[], byte[]>(byteFamily, byteQualifier));
+ result.add(Pair.newPair(byteFamily, byteQualifier));
}
colBlkIndex++;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 53b690c..592451f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -52,7 +52,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- // check whether this is a TopN query
+ // allow custom measures hack
notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
@@ -86,10 +86,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
boolean isExactAggregation = isExactAggregation(cuboid, groups, filterDimsD, singleValuesD, derivedPostAggregation);
context.setExactAggregation(isExactAggregation);
- if (isExactAggregation) {
- metrics = replaceHolisticCountDistinct(metrics);
- }
-
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
TupleFilter filterD = translateDerived(filter, groupsD);
@@ -111,22 +107,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
if (scanners.isEmpty())
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
- return newSequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
- }
-
- private ITupleIterator newSequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics, TupleInfo returnTupleInfo, StorageContext context) {
- TblColRef topNCol = null;
- for (FunctionDesc func : metrics) {
- if (func.isTopN()) {
- topNCol = func.getTopNLiteralColumn();
- break;
- }
- }
-
- if (topNCol != null)
- return new SequentialCubeTopNTupleIterator(scanners, cuboid, dimensionsD, topNCol, metrics, returnTupleInfo, context);
- else
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
}
private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -248,27 +229,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return exact;
}
- private Set<FunctionDesc> replaceHolisticCountDistinct(Set<FunctionDesc> metrics) {
- // for count distinct, try use its holistic version if possible
- Set<FunctionDesc> result = new LinkedHashSet<FunctionDesc>();
- for (FunctionDesc metric : metrics) {
- if (metric.isCountDistinct() == false) {
- result.add(metric);
- continue;
- }
-
- FunctionDesc holisticVersion = null;
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc measureFunc = measure.getFunction();
- if (measureFunc.equals(metric) && measureFunc.isHolisticCountDistinct()) {
- holisticVersion = measureFunc;
- }
- }
- result.add(holisticVersion == null ? metric : holisticVersion);
- }
- return result;
- }
-
@SuppressWarnings("unchecked")
private TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
if (filter == null)
@@ -343,15 +303,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
private void setThreshold(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, StorageContext context) {
- boolean hasMemHungryCountDistinct = false;
+ boolean hasMemHungryMeasure = false;
for (FunctionDesc func : metrics) {
- if (func.isCountDistinct() && !func.isHolisticCountDistinct()) {
- hasMemHungryCountDistinct = true;
- }
+ hasMemHungryMeasure |= func.getMeasureType().isMemoryHungry();
}
- // need to limit the memory usage for memory hungry count distinct
- if (hasMemHungryCountDistinct == false) {
+ // need to limit the memory usage for memory hungry measures
+ if (hasMemHungryMeasure == false) {
return;
}
@@ -380,6 +338,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
}
+ private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+ for (MeasureDesc measure : cubeDesc.getMeasures()) {
+ MeasureType<?> measureType = measure.getFunction().getMeasureType();
+ measureType.beforeStorageQuery(measure, sqlDigest);
+ }
+ }
+
// ============================================================================
@Override
@@ -397,10 +362,4 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return false;
}
- private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- MeasureType measureType = measure.getFunction().getMeasureType();
- measureType.beforeStorageQuery(measure, sqlDigest);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index 3653af1..a7346af 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -18,17 +18,12 @@
package org.apache.kylin.storage.hbase.cube.v2;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -60,24 +55,19 @@ public class CubeTupleConverter {
final int[] gtColIdx;
final int[] tupleIdx;
final Object[] gtValues;
- final MeasureType[] measureTypes;
+ final MeasureType<?>[] measureTypes;
final List<IAdvMeasureFiller> advMeasureFillers;
final List<Integer> advMeasureIndexInGTValues;
final int nSelectedDims;
- final TblColRef topNCol;
- int topNColTupleIdx;
- int topNMeasureTupleIdx;
- Dictionary<String> topNColDict;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, TblColRef topNCol) {
+ Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
this.tupleInfo = returnTupleInfo;
this.derivedColFillers = Lists.newArrayList();
- this.topNCol = topNCol;
List<TblColRef> cuboidDims = cuboid.getColumns();
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
@@ -117,7 +107,7 @@ public class CubeTupleConverter {
tupleIdx[iii] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
}
- MeasureType measureType = metric.getMeasureType();
+ MeasureType<?> measureType = metric.getMeasureType();
if (measureType.needAdvancedTupleFilling()) {
Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
@@ -129,14 +119,6 @@ public class CubeTupleConverter {
iii++;
}
- if (this.topNCol != null) {
- this.topNColTupleIdx = tupleInfo.hasColumn(this.topNCol) ? tupleInfo.getColumnIndex(this.topNCol) : -1;
- // topN only allow 1 measure
- this.topNMeasureTupleIdx = tupleIdx[tupleIdx.length - 1];
-
- this.topNColDict = (Dictionary<String>)cubeSeg.getDictionary(this.topNCol);
- }
-
// prepare derived columns and filler
Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboidDims, null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
@@ -196,13 +178,6 @@ public class CubeTupleConverter {
}
}
- public Iterator<Tuple> translateTopNResult(GTRecord record, Tuple tuple) {
- translateResult(record, tuple);
- Object topNCounterObj = tuple.getAllValues()[topNMeasureTupleIdx];
- assert (topNCounterObj instanceof TopNCounter);
- return new TopNCounterTupleIterator(tuple, (TopNCounter) topNCounterObj);
- }
-
private interface IDerivedColumnFiller {
public void fillDerivedColumns(Object[] gtValues, Tuple tuple);
}
@@ -292,38 +267,4 @@ public class CubeTupleConverter {
private static String toString(Object o) {
return o == null ? null : o.toString();
}
-
- private class TopNCounterTupleIterator implements Iterator {
-
- private Tuple tuple;
- private Iterator<Counter> topNCounterIterator;
- private Counter<ByteArray> counter;
-
- private TopNCounterTupleIterator(Tuple tuple, TopNCounter topNCounter) {
- this.tuple = tuple;
- this.topNCounterIterator = topNCounter.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return topNCounterIterator.hasNext();
- }
-
- @Override
- public Tuple next() {
- counter = topNCounterIterator.next();
- int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
- String colValue = topNColDict.getValueFromId(key);
- tuple.setDimensionValue(topNColTupleIdx, colValue);
- tuple.setMeasureValue(topNMeasureTupleIdx, counter.getCount());
-
- return tuple;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 303c360..ff7498b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -26,7 +26,6 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
deleted file mode 100644
index 9eeae4e..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package org.apache.kylin.storage.hbase.cube.v2;
-
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.Tuple;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.StorageContext;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Created by shaoshi on 10/28/15.
- */
-public class SequentialCubeTopNTupleIterator extends SequentialCubeTupleIterator {
-
- private Iterator<Tuple> innerResultIterator;
- private TblColRef topNCol;
-
- public SequentialCubeTopNTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
- TblColRef topNCol, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
-
- super(scanners, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context);
- this.topNCol = topNCol;
- }
-
- @Override
- public boolean hasNext() {
- if (next != null)
- return true;
- if (innerResultIterator == null) {
- if (curScanner == null) {
- if (scannerIterator.hasNext()) {
- curScanner = scannerIterator.next();
- curRecordIterator = curScanner.iterator();
- curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo, topNCol);
- } else {
- return false;
- }
- }
-
- if (curRecordIterator.hasNext()) {
- innerResultIterator = curTupleConverter.translateTopNResult(curRecordIterator.next(), tuple);
- return hasNext();
- } else {
- close(curScanner);
- curScanner = null;
- curRecordIterator = null;
- curTupleConverter = null;
- innerResultIterator = null;
- return hasNext();
- }
-
- }
-
-
- if (innerResultIterator.hasNext()) {
- next = innerResultIterator.next();
- return true;
- } else {
- innerResultIterator = null;
- return hasNext();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index 5bf5e95..7812baf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -75,7 +75,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
if (scannerIterator.hasNext()) {
curScanner = scannerIterator.next();
curRecordIterator = curScanner.iterator();
- curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo, null);
+ curTupleConverter = new CubeTupleConverter(curScanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
} else {
return false;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 036d9b8..10e80ae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -30,6 +30,7 @@ import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.hllc.HLLCMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -78,7 +79,7 @@ public class EndpointAggregators {
} else {
int index = tableInfo.findFactTableColumn(functionDesc.getParameter().getValue());
Preconditions.checkState(index >= 0, "Column " + functionDesc.getParameter().getValue() + " is not found in II");
- if (functionDesc.isCountDistinct()) {
+ if (HLLCMeasureType.isCountDistinct(functionDesc)) {
return new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision());
} else {
return new MetricInfo(MetricType.Normal, index);
@@ -144,10 +145,10 @@ public class EndpointAggregators {
MeasureAggregator[] aggrs = new MeasureAggregator[funcNames.length];
for (int i = 0; i < aggrs.length; i++) {
if (metricInfos[i].type == MetricType.DistinctCount) {
- aggrs[i] = MeasureAggregator.create(funcNames[i], dataTypes[i]);
+ aggrs[i] = MeasureAggregator.create(funcNames[i], DataType.getType(dataTypes[i]));
} else {
//all other fixed length measures can be aggregated as long
- aggrs[i] = MeasureAggregator.create(funcNames[i], "long");
+ aggrs[i] = MeasureAggregator.create(funcNames[i], DataType.getType("long"));
}
}
return aggrs;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index e638386..4ec421b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -42,6 +42,7 @@ import org.apache.kylin.common.util.RangeUtil;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.measure.hllc.HLLCMeasureType;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -205,7 +206,7 @@ public class EndpointTupleIterator implements ITupleIterator {
boolean updated = false;
for (TblColRef column : columns) {
if (column.isSameAs(factTableName, functionDesc.getParameter().getValue())) {
- if (functionDesc.isCountDistinct()) {
+ if (HLLCMeasureType.isCountDistinct(functionDesc)) {
//TODO: default precision might need be configurable
String iiDefaultHLLC = "hllc10";
functionDesc.setReturnType(iiDefaultHLLC);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index a7d0776..c798289 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -374,12 +374,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
int space = 0;
for (MeasureDesc measureDesc : cubeSegment.getCubeDesc().getMeasures()) {
DataType returnType = measureDesc.getFunction().getReturnDataType();
- if (returnType.isHLLC()) {
- // for HLL, it will be compressed when export to bytes
- space += returnType.getStorageBytesEstimate() * 0.75;
- } else {
- space += returnType.getStorageBytesEstimate();
- }
+ space += returnType.getStorageBytesEstimate();
}
bytesLength += space;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
index 205c0db..e565e86 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
@@ -152,7 +152,7 @@ public class HBaseMROutput2 implements IMROutput2 {
this.parsedKey = new ByteArrayWritable();
this.parsedValue = new Object[measuresDescs.size()];
- this.parsedPair = new Pair<ByteArrayWritable, Object[]>(parsedKey, parsedValue);
+ this.parsedPair = Pair.newPair(parsedKey, parsedValue);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index f190186..dff1e51 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -241,7 +241,7 @@ public class HBaseMROutput2Transition implements IMROutput2 {
if (parsedPair == null) {
parsedKey = new ByteArrayWritable();
parsedValue = new Object[seg.getCubeDesc().getMeasures().size()];
- parsedPair = new Pair<ByteArrayWritable, Object[]>(parsedKey, parsedValue);
+ parsedPair = Pair.newPair(parsedKey, parsedValue);
}
// merge by cuboid files
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index 617af76..59a5fed 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -122,34 +122,21 @@ public class RowValueDecoder implements Cloneable {
}
}
- public boolean hasMemHungryCountDistinct() {
+ public boolean hasMemHungryMeasures() {
for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
FunctionDesc func = measures[i].getFunction();
- if (func.isCountDistinct() && !func.isHolisticCountDistinct()) {
+ if (func.getMeasureType().isMemoryHungry())
return true;
- }
}
return false;
}
- public static boolean hasMemHungryCountDistinct(Collection<RowValueDecoder> rowValueDecoders) {
+ public static boolean hasMemHungryMeasures(Collection<RowValueDecoder> rowValueDecoders) {
for (RowValueDecoder decoder : rowValueDecoders) {
- if (decoder.hasMemHungryCountDistinct())
+ if (decoder.hasMemHungryMeasures())
return true;
}
return false;
}
- public static MeasureDesc findTopN(Collection<RowValueDecoder> rowValueDecoders) {
- for (RowValueDecoder decoder : rowValueDecoders) {
- for (int i = decoder.projectionIndex.nextSetBit(0); i >= 0; i = decoder.projectionIndex.nextSetBit(i + 1)) {
- MeasureDesc measure = decoder.measures[i];
- FunctionDesc func = measure.getFunction();
- if (func.isTopN())
- return measure;
- }
- }
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index e71089a..8a88b6d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -325,7 +325,7 @@ public class GridTableHBaseBenchmark {
int logicRows = nRows / nColumns;
for (int i = 0; i < nColumns; i++) {
if (rand.nextDouble() < hitRatio) {
- hitsForColumnScan.add(new Pair<Integer, Integer>(i * logicRows, (i + 1) * logicRows));
+ hitsForColumnScan.add(Pair.newPair(i * logicRows, (i + 1) * logicRows));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index f717d82..1d85922 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -167,7 +167,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
}
buf.putInt(i);
- Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
+ Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
list.add(pair);
}
@@ -200,7 +200,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
}
buf.putInt(i * 2);
- Pair<byte[], byte[]> pair = new Pair<byte[], byte[]>(fuzzyKey, mask);
+ Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
list.add(pair);
}
@@ -309,8 +309,8 @@ public class TestFuzzyRowFilterV2EndToEnd {
byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
- Pair<byte[], byte[]> pair1 = new Pair<byte[], byte[]>(fuzzyKey1, mask1);
- Pair<byte[], byte[]> pair2 = new Pair<byte[], byte[]>(fuzzyKey2, mask2);
+ Pair<byte[], byte[]> pair1 = Pair.newPair(fuzzyKey1, mask1);
+ Pair<byte[], byte[]> pair2 = Pair.newPair(fuzzyKey2, mask2);
FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(Lists.newArrayList(pair1));
FuzzyRowFilterV2 fuzzyRowFilter2 = new FuzzyRowFilterV2(Lists.newArrayList(pair2));
[14/21] kylin git commit: KYLIN-976 extract
MeasureIngester.reEncodeDictionary()
Posted by li...@apache.org.
KYLIN-976 extract MeasureIngester.reEncodeDictionary()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f011d10f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f011d10f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f011d10f
Branch: refs/heads/2.x-staging
Commit: f011d10fff9ac9224411dcab38bdbefe9a2f54f3
Parents: 9939682
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Nov 27 16:59:51 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:52:10 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/measure/MeasureIngester.java | 2 +
.../org/apache/kylin/measure/MeasureType.java | 3 -
.../kylin/measure/basic/BasicMeasureType.java | 7 --
.../kylin/measure/basic/BigDecimalIngester.java | 6 ++
.../kylin/measure/basic/DoubleIngester.java | 6 ++
.../kylin/measure/basic/LongIngester.java | 6 ++
.../kylin/measure/hllc/HLLCMeasureType.java | 11 ++-
.../kylin/measure/topn/TopNMeasureType.java | 43 +++++++++---
.../mr/steps/MergeCuboidFromStorageMapper.java | 68 +++++++------------
.../engine/mr/steps/MergeCuboidMapper.java | 71 +++++++-------------
10 files changed, 107 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
index 8d6e601..9c7b406 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -41,4 +41,6 @@ abstract public class MeasureIngester<V> {
}
abstract public V valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap);
+
+ abstract public V reEncodeDictionary(V value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 604a7b6..4fe59c0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -21,7 +21,6 @@ package org.apache.kylin.measure;
import java.util.List;
import java.util.Map;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.basic.BasicMeasureFactory;
import org.apache.kylin.measure.hllc.HLLCAggregationFactory;
import org.apache.kylin.measure.topn.TopNMeasureFactory;
@@ -86,8 +85,6 @@ abstract public class MeasureType {
abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
- abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
-
/* ============================================================================
* Cube Selection
* ---------------------------------------------------------------------------- */
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index f6bf090..fe53bab 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -20,7 +20,6 @@ package org.apache.kylin.measure.basic;
import java.util.List;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
@@ -126,10 +125,4 @@ public class BasicMeasureType extends MeasureType {
return null;
}
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
index bb743d6..ea1495c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
@@ -28,6 +28,7 @@ import org.apache.kylin.metadata.model.TblColRef;
public class BigDecimalIngester extends MeasureIngester<BigDecimal> {
+ @Override
public BigDecimal valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
@@ -37,4 +38,9 @@ public class BigDecimalIngester extends MeasureIngester<BigDecimal> {
else
return new BigDecimal(values[0]);
}
+
+ @Override
+ public BigDecimal reEncodeDictionary(BigDecimal value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index 506ed19..aaa754a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -31,6 +31,7 @@ public class DoubleIngester extends MeasureIngester<DoubleMutable> {
// avoid repeated object creation
private DoubleMutable current = new DoubleMutable();
+ @Override
public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
@@ -42,4 +43,9 @@ public class DoubleIngester extends MeasureIngester<DoubleMutable> {
l.set(Double.parseDouble(values[0]));
return l;
}
+
+ @Override
+ public DoubleMutable reEncodeDictionary(DoubleMutable value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index 5bf1257..bdc1704 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -31,6 +31,7 @@ public class LongIngester extends MeasureIngester<LongMutable> {
// avoid repeated object creation
private LongMutable current = new LongMutable();
+ @Override
public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
@@ -42,4 +43,9 @@ public class LongIngester extends MeasureIngester<LongMutable> {
l.set(Long.parseLong(values[0]));
return l;
}
+
+ @Override
+ public LongMutable reEncodeDictionary(LongMutable value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index ee90818..2ad7630 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -74,6 +74,11 @@ public class HLLCMeasureType extends MeasureType {
hllc.add(v == null ? "__nUlL__" : v);
return hllc;
}
+
+ @Override
+ public HyperLogLogPlusCounter reEncodeDictionary(HyperLogLogPlusCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
+ throw new UnsupportedOperationException();
+ }
};
}
@@ -91,10 +96,4 @@ public class HLLCMeasureType extends MeasureType {
return null;
}
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 1d2c87b..561f9f1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,9 +18,11 @@
package org.apache.kylin.measure.topn;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
@@ -88,6 +90,37 @@ public class TopNMeasureType extends MeasureType {
topNCounter.offer(key, counter);
return topNCounter;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TopNCounter reEncodeDictionary(TopNCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
+ TopNCounter<ByteArray> topNCounter = (TopNCounter<ByteArray>) value;
+
+ TblColRef colRef = measureDesc.getFunction().getTopNLiteralColumn();
+ Dictionary<String> sourceDict = oldDicts.get(colRef);
+ Dictionary<String> mergedDict = newDicts.get(colRef);
+
+ int topNSize = topNCounter.size();
+ byte[] newIdBuf = new byte[topNSize * mergedDict.getSizeOfId()];
+ byte[] literal = new byte[sourceDict.getSizeOfValue()];
+
+ int bufOffset = 0;
+ for (Counter<ByteArray> c : topNCounter) {
+ int oldId = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(), c.getItem().length());
+ int newId;
+ int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
+ if (size < 0) {
+ newId = mergedDict.nullId();
+ } else {
+ newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ }
+
+ BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
+ c.getItem().set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
+ bufOffset += mergedDict.getSizeOfId();
+ }
+ return value;
+ }
};
}
@@ -98,14 +131,8 @@ public class TopNMeasureType extends MeasureType {
@Override
public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
+ TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(1);
+ return Collections.singletonList(literalCol);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 4d37089..ddcc75c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -22,10 +22,9 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -48,6 +47,8 @@ import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -60,6 +61,7 @@ import com.google.common.collect.Lists;
/**
* @author shaoshi
*/
+@SuppressWarnings({ "rawtypes", "unchecked" })
public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, ByteArrayWritable, ByteArrayWritable> {
private static final Logger logger = LoggerFactory.getLogger(MergeCuboidFromStorageMapper.class);
@@ -87,7 +89,9 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
private MeasureCodec codec;
private ByteArrayWritable outputValue = new ByteArrayWritable();
- private List<Integer> topNMeasureIdx;
+ private List<Pair<Integer, MeasureIngester>> dictMeasures;
+ private Map<TblColRef, Dictionary<String>> oldDicts;
+ private Map<TblColRef, Dictionary<String>> newDicts;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
@@ -115,12 +119,18 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
measureDescs = cubeDesc.getMeasures();
codec = new MeasureCodec(measureDescs);
- topNMeasureIdx = Lists.newArrayList();
+ dictMeasures = Lists.newArrayList();
for (int i = 0; i < measureDescs.size(); i++) {
- if (measureDescs.get(i).getFunction().isTopN()) {
- topNMeasureIdx.add(i);
+ MeasureDesc measureDesc = measureDescs.get(i);
+ MeasureType measureType = MeasureType.create(measureDesc.getFunction());
+ if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
+ if (dictMeasures.size() > 0) {
+ oldDicts = sourceCubeSegment.buildDictionaryMap();
+ newDicts = mergedCubeSegment.buildDictionaryMap();
+ }
}
@Override
@@ -193,7 +203,9 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
outputKey.set(newKeyBuf.array(), 0, fullKeySize);
// re-encode measures if dictionary is used
- reEncodeMeasure(value);
+ if (dictMeasures.size() > 0) {
+ reEncodeMeasure(value);
+ }
valueBuf.clear();
codec.encode(value, valueBuf);
@@ -217,45 +229,11 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
}
}
- @SuppressWarnings("unchecked")
private void reEncodeMeasure(Object[] measureObjs) throws IOException, InterruptedException {
- // currently only topN uses dictionary in measure obj
- if (topNMeasureIdx.isEmpty())
- return;
-
- int bufOffset = 0;
- for (int idx : topNMeasureIdx) {
- TopNCounter<ByteArray> topNCounters = (TopNCounter<ByteArray>) measureObjs[idx];
-
- MeasureDesc measureDesc = measureDescs.get(idx);
- TblColRef colRef = measureDesc.getFunction().getTopNLiteralColumn();
- DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(colRef));
- Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(colRef));
-
- int topNSize = topNCounters.size();
- while (sourceDict.getSizeOfValue() * topNSize > newKeyBodyBuf.length - bufOffset || //
- mergedDict.getSizeOfValue() * topNSize > newKeyBodyBuf.length - bufOffset || //
- mergedDict.getSizeOfId() * topNSize > newKeyBodyBuf.length - bufOffset) {
- byte[] oldBuf = newKeyBodyBuf;
- newKeyBodyBuf = new byte[2 * newKeyBodyBuf.length];
- System.arraycopy(oldBuf, 0, newKeyBodyBuf, 0, oldBuf.length);
- }
-
- for (Counter<ByteArray> c : topNCounters) {
- int idInSourceDict = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(), c.getItem().length());
- int idInMergedDict;
- int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
- if (size < 0) {
- idInMergedDict = mergedDict.nullId();
- } else {
- idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBodyBuf, bufOffset, size);
- }
-
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
- c.getItem().set(newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
- bufOffset += mergedDict.getSizeOfId();
- }
+ for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
+ int i = pair.getFirst();
+ MeasureIngester ingester = pair.getSecond();
+ measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f011d10f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index c596318..c4e1ff5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -22,17 +22,17 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.SplittedBytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -48,6 +48,8 @@ import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -57,6 +59,7 @@ import com.google.common.collect.Lists;
/**
* @author ysong1, honma
*/
+@SuppressWarnings({"rawtypes", "unchecked"})
public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
private KylinConfig config;
@@ -78,7 +81,9 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
private HashMap<TblColRef, Boolean> dimensionsNeedDict = new HashMap<TblColRef, Boolean>();
// for re-encode measures that use dictionary
- private List<Integer> topNMeasureIdx;
+ private List<Pair<Integer, MeasureIngester>> dictMeasures;
+ private Map<TblColRef, Dictionary<String>> oldDicts;
+ private Map<TblColRef, Dictionary<String>> newDicts;
private List<MeasureDesc> measureDescs;
private MeasureCodec codec;
private Object[] measureObjs;
@@ -115,12 +120,19 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
measureObjs = new Object[measureDescs.size()];
valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
outputValue = new Text();
- topNMeasureIdx = Lists.newArrayList();
+
+ dictMeasures = Lists.newArrayList();
for (int i = 0; i < measureDescs.size(); i++) {
- if (measureDescs.get(i).getFunction().isTopN()) {
- topNMeasureIdx.add(i);
+ MeasureDesc measureDesc = measureDescs.get(i);
+ MeasureType measureType = MeasureType.create(measureDesc.getFunction());
+ if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
+ if (dictMeasures.size() > 0) {
+ oldDicts = sourceCubeSegment.buildDictionaryMap();
+ newDicts = mergedCubeSegment.buildDictionaryMap();
+ }
}
private static final Pattern JOB_NAME_PATTERN = Pattern.compile("kylin-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})");
@@ -214,9 +226,13 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
outputKey.set(newKeyBuf.array(), 0, fullKeySize);
// re-encode measures if dictionary is used
- if (topNMeasureIdx.size() > 0) {
+ if (dictMeasures.size() > 0) {
codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
- reEncodeMeasure(measureObjs);
+ for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
+ int i = pair.getFirst();
+ MeasureIngester ingester = pair.getSecond();
+ measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
+ }
valueBuf.clear();
codec.encode(measureObjs, valueBuf);
outputValue.set(valueBuf.array(), 0, valueBuf.position());
@@ -240,43 +256,4 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
return ret;
}
}
-
- @SuppressWarnings("unchecked")
- private void reEncodeMeasure(Object[] measureObjs) throws IOException, InterruptedException {
- int bufOffset = 0;
- for (int idx : topNMeasureIdx) {
- // only TopN measure uses dic
- TopNCounter<ByteArray> topNCounters = (TopNCounter<ByteArray>) measureObjs[idx];
-
- MeasureDesc measureDesc = measureDescs.get(idx);
- TblColRef colRef = measureDesc.getFunction().getTopNLiteralColumn();
- DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(colRef));
- Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(colRef));
-
- int topNSize = topNCounters.size();
- while (sourceDict.getSizeOfValue() * topNSize > newKeyBodyBuf.length - bufOffset || //
- mergedDict.getSizeOfValue() * topNSize > newKeyBodyBuf.length - bufOffset || //
- mergedDict.getSizeOfId() * topNSize > newKeyBodyBuf.length - bufOffset) {
- byte[] oldBuf = newKeyBodyBuf;
- newKeyBodyBuf = new byte[2 * newKeyBodyBuf.length];
- System.arraycopy(oldBuf, 0, newKeyBodyBuf, 0, oldBuf.length);
- }
-
- for (Counter<ByteArray> c : topNCounters) {
- int idInSourceDict = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(), c.getItem().length());
- int idInMergedDict;
- int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
- if (size < 0) {
- idInMergedDict = mergedDict.nullId();
- } else {
- idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBodyBuf, bufOffset, size);
- }
-
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
- c.getItem().set(newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
- bufOffset += mergedDict.getSizeOfId();
- }
- }
- }
}
[15/21] kylin git commit: KYLIN-976 Extract Rewrite related methdos
Posted by li...@apache.org.
KYLIN-976 Extract Rewrite related methdos
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/98a3e4c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/98a3e4c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/98a3e4c4
Branch: refs/heads/2.x-staging
Commit: 98a3e4c4522ed9f1edf4c3e4488ba5896ee5d0d9
Parents: f4c4c88
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Dec 1 11:39:31 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:52:10 2015 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../apache/kylin/measure/MeasureIngester.java | 6 +-
.../org/apache/kylin/measure/MeasureType.java | 10 +-
.../kylin/measure/basic/BasicMeasureType.java | 11 ++
.../kylin/measure/basic/BigDecimalIngester.java | 5 -
.../kylin/measure/basic/DoubleIngester.java | 5 -
.../kylin/measure/basic/LongIngester.java | 5 -
.../kylin/measure/hllc/HLLCMeasureType.java | 16 +-
.../measure/hllc/HLLDistinctCountAggFunc.java | 153 +++++++++++++++++++
.../kylin/measure/topn/TopNMeasureType.java | 10 ++
.../kylin/metadata/model/FunctionDesc.java | 39 +++--
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 13 +-
.../apache/kylin/query/schema/OLAPTable.java | 10 +-
.../query/sqlfunc/HLLDistinctCountAggFunc.java | 152 ------------------
17 files changed, 237 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 0c1b3c9..624bb0b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -191,7 +191,7 @@ public class CubeCapabilityChecker {
if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
break;
- MeasureType measureType = MeasureType.create(measure.getFunction());
+ MeasureType measureType = measure.getFunction().getMeasureType();
if (measureType instanceof BasicMeasureType)
continue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 6124ac9..3c91da4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -766,7 +766,7 @@ public class CubeDesc extends RootPersistentEntity {
}
for (MeasureDesc measure : measures) {
- MeasureType aggrType = MeasureType.create(measure.getFunction());
+ MeasureType aggrType = measure.getFunction().getMeasureType();
result.addAll(aggrType.getColumnsNeedDictionary(measure));
}
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
index 9c7b406..bc387fe 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -28,7 +28,7 @@ import org.apache.kylin.metadata.model.TblColRef;
abstract public class MeasureIngester<V> {
public static MeasureIngester<?> create(MeasureDesc measure) {
- return MeasureType.create(measure.getFunction()).newIngester();
+ return measure.getFunction().getMeasureType().newIngester();
}
public static MeasureIngester<?>[] create(Collection<MeasureDesc> measures) {
@@ -42,5 +42,7 @@ abstract public class MeasureIngester<V> {
abstract public V valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap);
- abstract public V reEncodeDictionary(V value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts);
+ public V reEncodeDictionary(V value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 8e7de6f..0891d1e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -45,10 +45,6 @@ abstract public class MeasureType {
factoryRegistry.put(FunctionDesc.FUNC_TOP_N, new TopNMeasureFactory());
}
- public static MeasureType create(FunctionDesc function) {
- return create(function.getExpression(), function.getReturnType());
- }
-
public static MeasureType create(String funcName, String dataType) {
funcName = funcName.toUpperCase();
dataType = dataType.toLowerCase();
@@ -102,6 +98,12 @@ abstract public class MeasureType {
* Query
* ---------------------------------------------------------------------------- */
+ // TODO support user defined calcite aggr function
+
+ abstract public boolean needRewrite();
+
+ abstract public Class<?> getRewriteAggregationFunctionClass();
+
/* ============================================================================
* Storage
* ---------------------------------------------------------------------------- */
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index f314870..a6b36bb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -115,5 +115,16 @@ public class BasicMeasureType extends MeasureType {
private boolean isMin() {
return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
}
+
+ @Override
+ public boolean needRewrite() {
+ return !isSum();
+ }
+
+ @Override
+ public Class<?> getRewriteAggregationFunctionClass() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
index ea1495c..721ba00 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
@@ -38,9 +38,4 @@ public class BigDecimalIngester extends MeasureIngester<BigDecimal> {
else
return new BigDecimal(values[0]);
}
-
- @Override
- public BigDecimal reEncodeDictionary(BigDecimal value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
- throw new UnsupportedOperationException();
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index aaa754a..70ca727 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -43,9 +43,4 @@ public class DoubleIngester extends MeasureIngester<DoubleMutable> {
l.set(Double.parseDouble(values[0]));
return l;
}
-
- @Override
- public DoubleMutable reEncodeDictionary(DoubleMutable value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
- throw new UnsupportedOperationException();
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index bdc1704..2547162 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -43,9 +43,4 @@ public class LongIngester extends MeasureIngester<LongMutable> {
l.set(Long.parseLong(values[0]));
return l;
}
-
- @Override
- public LongMutable reEncodeDictionary(LongMutable value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
- throw new UnsupportedOperationException();
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 4a73478..fd71c00 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -73,11 +73,6 @@ public class HLLCMeasureType extends MeasureType {
hllc.add(v == null ? "__nUlL__" : v);
return hllc;
}
-
- @Override
- public HyperLogLogPlusCounter reEncodeDictionary(HyperLogLogPlusCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDict, Map<TblColRef, Dictionary<String>> newDict) {
- throw new UnsupportedOperationException();
- }
};
}
@@ -89,4 +84,15 @@ public class HLLCMeasureType extends MeasureType {
return new LDCAggregator();
}
+ @Override
+ public boolean needRewrite() {
+ return true;
+ }
+
+ @Override
+ public Class<?> getRewriteAggregationFunctionClass() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
new file mode 100644
index 0000000..b00da5f
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author xjiang
+ */
+public class HLLDistinctCountAggFunc {
+
+ private static final Logger logger = LoggerFactory.getLogger(HLLDistinctCountAggFunc.class);
+
+ public static HyperLogLogPlusCounter init() {
+ return null;
+ }
+
+ public static HyperLogLogPlusCounter initAdd(Object v) {
+ if (v instanceof Long) { // holistic case
+ long l = (Long) v;
+ return new FixedValueHLLCMockup(l);
+ } else {
+ HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
+ return new HyperLogLogPlusCounter(c);
+ }
+ }
+
+ public static HyperLogLogPlusCounter add(HyperLogLogPlusCounter counter, Object v) {
+ if (v instanceof Long) { // holistic case
+ long l = (Long) v;
+ if (counter == null) {
+ return new FixedValueHLLCMockup(l);
+ } else {
+ if (!(counter instanceof FixedValueHLLCMockup))
+ throw new IllegalStateException("counter is not FixedValueHLLCMockup");
+
+ ((FixedValueHLLCMockup) counter).set(l);
+ return counter;
+ }
+ } else {
+ HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
+ if (counter == null) {
+ return new HyperLogLogPlusCounter(c);
+ } else {
+ counter.merge(c);
+ return counter;
+ }
+ }
+ }
+
+ public static HyperLogLogPlusCounter merge(HyperLogLogPlusCounter counter0, Object counter1) {
+ return add(counter0, counter1);
+ }
+
+ public static long result(HyperLogLogPlusCounter counter) {
+ return counter == null ? 0L : counter.getCountEstimate();
+ }
+
+ @SuppressWarnings("serial")
+ private static class FixedValueHLLCMockup extends HyperLogLogPlusCounter {
+
+ private Long value = null;
+
+ FixedValueHLLCMockup(long value) {
+ this.value = value;
+ }
+
+ public void set(long value) {
+ if (this.value == null) {
+ this.value = value;
+ } else {
+ long oldValue = Math.abs(this.value.longValue());
+ long take = Math.max(oldValue, value);
+ logger.warn("Error to aggregate holistic count distinct, old value " + oldValue + ", new value " + value + ", taking " + take);
+ this.value = -take; // make it obvious that this value is wrong
+ }
+ }
+
+ @Override
+ public void clear() {
+ this.value = null;
+ }
+
+ @Override
+ protected void add(long hash) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void merge(HyperLogLogPlusCounter another) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getCountEstimate() {
+ return value;
+ }
+
+ @Override
+ public void writeRegisters(ByteBuffer out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void readRegisters(ByteBuffer in) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + (int) (value ^ (value >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!super.equals(obj))
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ FixedValueHLLCMockup other = (FixedValueHLLCMockup) obj;
+ if (!value.equals(other.value))
+ return false;
+ return true;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index d6c5a6f..77319be 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -167,4 +167,14 @@ public class TopNMeasureType extends MeasureType {
return null;
}
+ @Override
+ public boolean needRewrite() {
+ return false;
+ }
+
+ @Override
+ public Class<?> getRewriteAggregationFunctionClass() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 0c36873..17debda 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Collection;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -51,6 +52,7 @@ public class FunctionDesc {
private String returnType;
private DataType returnDataType;
+ private MeasureType measureType;
private boolean isDimensionAsMetric = false;
public void init(TableDesc factTable) {
@@ -77,6 +79,23 @@ public class FunctionDesc {
setReturnType(colRefs.get(0).getDatatype());
}
}
+
+ public MeasureType getMeasureType() {
+ if (isDimensionAsMetric)
+ return null;
+
+ if (measureType == null) {
+ measureType = MeasureType.create(getExpression(), getReturnType());
+ }
+ return measureType;
+ }
+
+ public boolean needRewrite() {
+ if (isDimensionAsMetric)
+ return false;
+
+ return getMeasureType().needRewrite();
+ }
public String getRewriteFieldName() {
if (isSum()) {
@@ -88,14 +107,19 @@ public class FunctionDesc {
}
}
- public boolean needRewrite() {
- return !isSum() && !isDimensionAsMetric() && !isTopN();
+ public DataType getRewriteFieldType() {
+ if (isCountDistinct() || isTopN())
+ return DataType.ANY;
+ else if (isSum() || isMax() || isMin())
+ return parameter.getColRefs().get(0).getType();
+ else
+ return returnDataType;
}
public ColumnDesc newFakeRewriteColumn(TableDesc sourceTable) {
ColumnDesc fakeCol = new ColumnDesc();
fakeCol.setName(getRewriteFieldName());
- fakeCol.setDatatype(getSQLType().toString());
+ fakeCol.setDatatype(getRewriteFieldType().toString());
if (isCount())
fakeCol.setNullable(false);
fakeCol.init(sourceTable);
@@ -179,15 +203,6 @@ public class FunctionDesc {
return count;
}
- public DataType getSQLType() {
- if (isCountDistinct() || isTopN())
- return DataType.ANY;
- else if (isSum() || isMax() || isMin())
- return parameter.getColRefs().get(0).getType();
- else
- return returnDataType;
- }
-
public String getReturnType() {
return returnType;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index ddcc75c..654f1b5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -122,7 +122,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
dictMeasures = Lists.newArrayList();
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
- MeasureType measureType = MeasureType.create(measureDesc.getFunction());
+ MeasureType measureType = measureDesc.getFunction().getMeasureType();
if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index c4e1ff5..b3f69ee 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -124,7 +124,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
dictMeasures = Lists.newArrayList();
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
- MeasureType measureType = MeasureType.create(measureDesc.getFunction());
+ MeasureType measureType = measureDesc.getFunction().getMeasureType();
if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index cbc0c56..9aa70ca 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -59,7 +59,6 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.query.sqlfunc.HLLDistinctCountAggFunc;
import com.google.common.base.Preconditions;
@@ -301,10 +300,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// rebuild function
RelDataType fieldType = aggCall.getType();
SqlAggFunction newAgg = aggCall.getAggregation();
- if (func.isCountDistinct()) {
- newAgg = createHyperLogLogAggFunction(fieldType);
- } else if (func.isCount()) {
+ if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
+ } else if (func.getMeasureType().getRewriteAggregationFunctionClass() != null) {
+ newAgg = createCustomAggFunction(fieldType, func.getMeasureType().getRewriteAggregationFunctionClass());
}
// rebuild aggregate call
@@ -312,10 +311,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
return newAggCall;
}
- private SqlAggFunction createHyperLogLogAggFunction(RelDataType returnType) {
+ private SqlAggFunction createCustomAggFunction(RelDataType returnType, Class<?> customAggFuncClz) {
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
- SqlIdentifier sqlIdentifier = new SqlIdentifier("HLL_COUNT", new SqlParserPos(1, 1));
- AggregateFunction aggFunction = AggregateFunctionImpl.create(HLLDistinctCountAggFunc.class);
+ SqlIdentifier sqlIdentifier = new SqlIdentifier(customAggFuncClz.getSimpleName(), new SqlParserPos(1, 1));
+ AggregateFunction aggFunction = AggregateFunctionImpl.create(customAggFuncClz);
List<RelDataType> argTypes = new ArrayList<RelDataType>();
List<SqlTypeFamily> typeFamilies = new ArrayList<SqlTypeFamily>();
for (FunctionParameter o : aggFunction.getParameters()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 94a91bf..a8789ea 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -184,11 +184,11 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
HashSet<ColumnDesc> updateColumns = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
- FunctionDesc functionDesc = m.getFunction();
- if (functionDesc.getReturnDataType() != functionDesc.getSQLType() && //
- functionDesc.getReturnDataType().isBigInt() && //
- functionDesc.getSQLType().isIntegerFamily()) {
- updateColumns.add(functionDesc.getParameter().getColRefs().get(0).getColumnDesc());
+ FunctionDesc func = m.getFunction();
+ if (func.getReturnDataType() != func.getRewriteFieldType() && //
+ func.getReturnDataType().isBigInt() && //
+ func.getRewriteFieldType().isIntegerFamily()) {
+ updateColumns.add(func.getParameter().getColRefs().get(0).getColumnDesc());
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/98a3e4c4/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
deleted file mode 100644
index 7881c42..0000000
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.sqlfunc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author xjiang
- */
-public class HLLDistinctCountAggFunc {
-
- private static final Logger logger = LoggerFactory.getLogger(HLLDistinctCountAggFunc.class);
-
- public static HyperLogLogPlusCounter init() {
- return null;
- }
-
- public static HyperLogLogPlusCounter initAdd(Object v) {
- if (v instanceof Long) { // holistic case
- long l = (Long) v;
- return new FixedValueHLLCMockup(l);
- } else {
- HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
- return new HyperLogLogPlusCounter(c);
- }
- }
-
- public static HyperLogLogPlusCounter add(HyperLogLogPlusCounter counter, Object v) {
- if (v instanceof Long) { // holistic case
- long l = (Long) v;
- if (counter == null) {
- return new FixedValueHLLCMockup(l);
- } else {
- if (!(counter instanceof FixedValueHLLCMockup))
- throw new IllegalStateException("counter is not FixedValueHLLCMockup");
-
- ((FixedValueHLLCMockup) counter).set(l);
- return counter;
- }
- } else {
- HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
- if (counter == null) {
- return new HyperLogLogPlusCounter(c);
- } else {
- counter.merge(c);
- return counter;
- }
- }
- }
-
- public static HyperLogLogPlusCounter merge(HyperLogLogPlusCounter counter0, Object counter1) {
- return add(counter0, counter1);
- }
-
- public static long result(HyperLogLogPlusCounter counter) {
- return counter == null ? 0L : counter.getCountEstimate();
- }
-
- private static class FixedValueHLLCMockup extends HyperLogLogPlusCounter {
-
- private Long value = null;
-
- FixedValueHLLCMockup(long value) {
- this.value = value;
- }
-
- public void set(long value) {
- if (this.value == null) {
- this.value = value;
- } else {
- long oldValue = Math.abs(this.value.longValue());
- long take = Math.max(oldValue, value);
- logger.warn("Error to aggregate holistic count distinct, old value " + oldValue + ", new value " + value + ", taking " + take);
- this.value = -take; // make it obvious that this value is wrong
- }
- }
-
- @Override
- public void clear() {
- this.value = null;
- }
-
- @Override
- protected void add(long hash) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void merge(HyperLogLogPlusCounter another) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getCountEstimate() {
- return value;
- }
-
- @Override
- public void writeRegisters(ByteBuffer out) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void readRegisters(ByteBuffer in) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + (int) (value ^ (value >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (!super.equals(obj))
- return false;
- if (getClass() != obj.getClass())
- return false;
- FixedValueHLLCMockup other = (FixedValueHLLCMockup) obj;
- if (!value.equals(other.value))
- return false;
- return true;
- }
- }
-
-}
[09/21] kylin git commit: KYLIN-976 Add ingester;
Build part done, in-mem cube test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
new file mode 100644
index 0000000..16563fa
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
+
+ final int precision;
+ HyperLogLogPlusCounter sum = null;
+
+ public HLLCAggregator(int precision) {
+ this.precision = precision;
+ }
+
+ @Override
+ public void reset() {
+ sum = null;
+ }
+
+ @Override
+ public void aggregate(HyperLogLogPlusCounter value) {
+ if (sum == null)
+ sum = new HyperLogLogPlusCounter(value);
+ else
+ sum.merge(value);
+ }
+
+ @Override
+ public HyperLogLogPlusCounter getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ // 1024 + 60 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 4 // precision
+ + 8 // ref to HLLC
+ + 8 // HLLC obj shell
+ + 32 + (1 << precision); // HLLC internal
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
new file mode 100644
index 0000000..ee90818
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class HLLCMeasureType extends MeasureType {
+
+ private final DataType dataType;
+
+ public HLLCMeasureType(DataType dataType) {
+ if ("hllc".equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+
+ this.dataType = dataType;
+
+ if (this.dataType.getPrecision() < 10 || this.dataType.getPrecision() > 16)
+ throw new IllegalArgumentException("HLLC precision must be between 10 and 16");
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ return HLLCSerializer.class;
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public MeasureIngester<?> newIngester() {
+ return new MeasureIngester<HyperLogLogPlusCounter>() {
+ HyperLogLogPlusCounter current = new HyperLogLogPlusCounter(dataType.getPrecision());
+
+ @Override
+ public HyperLogLogPlusCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ HyperLogLogPlusCounter hllc = current;
+ hllc.clear();
+ for (String v : values)
+ hllc.add(v == null ? "__nUlL__" : v);
+ return hllc;
+ }
+ };
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ if (dataType.isHLLC())
+ return new HLLCAggregator(dataType.getPrecision());
+ else
+ return new LDCAggregator();
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
new file mode 100644
index 0000000..7131201
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
+
+ private int precision;
+
+ public HLLCSerializer(DataType type) {
+ this.precision = type.getPrecision();
+ }
+
+ @Override
+ public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
+ try {
+ value.writeRegisters(out);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private HyperLogLogPlusCounter current() {
+ HyperLogLogPlusCounter hllc = current.get();
+ if (hllc == null) {
+ hllc = new HyperLogLogPlusCounter(precision);
+ current.set(hllc);
+ }
+ return hllc;
+ }
+
+ @Override
+ public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
+ HyperLogLogPlusCounter hllc = current();
+ try {
+ hllc.readRegisters(in);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return hllc;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return current().peekLength(in);
+ }
+
+ @Override
+ public int maxLength() {
+ return current().maxLength();
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return current().maxLength();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
new file mode 100644
index 0000000..5d96450
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.LongMutable;
+
+/**
+ * Long Distinct Count
+ */
+@SuppressWarnings("serial")
+public class LDCAggregator extends MeasureAggregator<LongMutable> {
+
+ private static LongMutable ZERO = new LongMutable(0);
+
+ private HLLCAggregator hllAgg = null;
+ private LongMutable state = new LongMutable(0);
+
+ @SuppressWarnings("rawtypes")
+ public void setDependentAggregator(MeasureAggregator agg) {
+ this.hllAgg = (HLLCAggregator) agg;
+ }
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ }
+
+ @Override
+ public LongMutable getState() {
+ if (hllAgg == null) {
+ return ZERO;
+ } else {
+ state.set(hllAgg.getState().getCountEstimate());
+ return state;
+ }
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
new file mode 100644
index 0000000..9b4c893
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+import java.util.Map;
+
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.measure.MeasureAggregator;
+
+import com.google.common.collect.Maps;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
+
+ int capacity = 0;
+ TopNCounter<ByteArray> sum = null;
+ Map<ByteArray, Double> sanityCheckMap;
+
+ @Override
+ public void reset() {
+ sum = null;
+ }
+
+ @Override
+ public void aggregate(TopNCounter<ByteArray> value) {
+ if (sum == null) {
+ capacity = value.getCapacity();
+ sum = new TopNCounter<>(capacity);
+ sanityCheckMap = Maps.newHashMap();
+ }
+ sum.merge(value);
+ }
+
+ @Override
+ public TopNCounter<ByteArray> getState() {
+
+ //sum.retain(capacity);
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return 8 * capacity / 4;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
new file mode 100644
index 0000000..b422316
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.kylin.common.topn.Counter;
+import org.apache.kylin.common.topn.DoubleDeltaSerializer;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
+/**
+ *
+ */
+public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteArray>> {
+
+ private DoubleDeltaSerializer dds = new DoubleDeltaSerializer(3);
+
+ private int precision;
+
+ public TopNCounterSerializer(DataType dataType) {
+ this.precision = dataType.getPrecision();
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ @SuppressWarnings("unused")
+ int capacity = in.getInt();
+ int size = in.getInt();
+ int keyLength = in.getInt();
+ dds.deserialize(in);
+ int len = in.position() - mark + keyLength * size;
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8);
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return precision * TopNCounter.EXTRA_SPACE_RATE * 8;
+ }
+
+ @Override
+ public void serialize(TopNCounter<ByteArray> value, ByteBuffer out) {
+ double[] counters = value.getCounters();
+ List<ByteArray> peek = value.peek(1);
+ int keyLength = peek.size() > 0 ? peek.get(0).length() : 0;
+ out.putInt(value.getCapacity());
+ out.putInt(value.size());
+ out.putInt(keyLength);
+ dds.serialize(counters, out);
+ Iterator<Counter<ByteArray>> iterator = value.iterator();
+ while (iterator.hasNext()) {
+ out.put(iterator.next().getItem().array());
+ }
+ }
+
+ @Override
+ public TopNCounter<ByteArray> deserialize(ByteBuffer in) {
+ int capacity = in.getInt();
+ int size = in.getInt();
+ int keyLength = in.getInt();
+ double[] counters = dds.deserialize(in);
+
+ TopNCounter<ByteArray> counter = new TopNCounter<ByteArray>(capacity);
+ ByteArray byteArray;
+ for (int i = 0; i < size; i++) {
+ byteArray = new ByteArray(keyLength);
+ in.get(byteArray.array());
+ counter.offerToHead(byteArray, counters[i]);
+ }
+
+ return counter;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
new file mode 100644
index 0000000..1ceb607
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+import org.apache.kylin.measure.IMeasureFactory;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+public class TopNMeasureFactory implements IMeasureFactory {
+
+ @Override
+ public MeasureType createMeasureType(String funcName, String dataType) {
+ if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName) == false)
+ throw new IllegalArgumentException();
+
+ return new TopNMeasureType(DataType.getType(dataType));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
new file mode 100644
index 0000000..1d2c87b
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.hllc.HLLCSerializer;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class TopNMeasureType extends MeasureType {
+
+ private final DataType dataType;
+
+ public TopNMeasureType(DataType dataType) {
+ if ("topn".equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+
+ this.dataType = dataType;
+
+ if (this.dataType.getPrecision() < 1 || this.dataType.getPrecision() > 1000)
+ throw new IllegalArgumentException("TopN precision must be between 1 and 1000");
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ return HLLCSerializer.class;
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public MeasureIngester<?> newIngester() {
+ return new MeasureIngester<TopNCounter>() {
+ @Override
+ public TopNCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (values.length != 2)
+ throw new IllegalArgumentException();
+
+ double counter = values[0] == null ? 0 : Double.parseDouble(values[0]);
+ String literal = values[1];
+
+ // encode literal using dictionary
+ TblColRef literalCol = measureDesc.getFunction().getTopNLiteralColumn();
+ Dictionary<String> dictionary = dictionaryMap.get(literalCol);
+ int keyEncodedValue = dictionary.getIdFromValue(literal);
+
+ ByteArray key = new ByteArray(dictionary.getSizeOfId());
+ BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, dictionary.getSizeOfId());
+
+ TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(dataType.getPrecision() * TopNCounter.EXTRA_SPACE_RATE);
+ topNCounter.offer(key, counter);
+ return topNCounter;
+ }
+ };
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ return new TopNAggregator();
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
new file mode 100644
index 0000000..134f305
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author yangli9
+ *
+ */
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
+
+ private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
+
+ final DataType type;
+ final int maxLength;
+
+ int avoidVerbose = 0;
+
+ public BigDecimalSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
+ this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
+ }
+
+ @Override
+ public void serialize(BigDecimal value, ByteBuffer out) {
+ if (value.scale() > type.getScale()) {
+ if (avoidVerbose % 10000 == 0) {
+ logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
+ }
+ value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
+ }
+ byte[] bytes = value.unscaledValue().toByteArray();
+ if (bytes.length + 2 > maxLength) {
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ BytesUtil.writeVInt(value.scale(), out);
+ BytesUtil.writeVInt(bytes.length, out);
+ out.put(bytes);
+ }
+
+ @Override
+ public BigDecimal deserialize(ByteBuffer in) {
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+
+ byte[] bytes = new byte[n];
+ in.get(bytes);
+
+ return new BigDecimal(new BigInteger(bytes), scale);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
new file mode 100644
index 0000000..235c99f
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DataType implements Serializable {
+
+ // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
+ public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
+ + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
+ + "|date|time|datetime|timestamp|byte|int|short|long|string|hllc|topn";
+
+ private static final String TYPE_PATTEN_TAIL = "\\s*" //
+ + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";
+
+ private static final Pattern TYPE_PATTERN = Pattern.compile( //
+ "(" + VALID_TYPES_STRING + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
+
+ public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
+ public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
+ public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
+ public static final Set<String> STRING_FAMILY = new HashSet<String>();
+ private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
+ static {
+ INTEGER_FAMILY.add("tinyint");
+ INTEGER_FAMILY.add("smallint");
+ INTEGER_FAMILY.add("integer");
+ INTEGER_FAMILY.add("bigint");
+
+ NUMBER_FAMILY.addAll(INTEGER_FAMILY);
+ NUMBER_FAMILY.add("float");
+ NUMBER_FAMILY.add("double");
+ NUMBER_FAMILY.add("decimal");
+ NUMBER_FAMILY.add("real");
+ NUMBER_FAMILY.add("numeric");
+
+ DATETIME_FAMILY.add("date");
+ DATETIME_FAMILY.add("time");
+ DATETIME_FAMILY.add("datetime");
+ DATETIME_FAMILY.add("timestamp");
+
+ STRING_FAMILY.add("varchar");
+ STRING_FAMILY.add("char");
+
+ LEGACY_TYPE_MAP.put("byte", "tinyint");
+ LEGACY_TYPE_MAP.put("int", "integer");
+ LEGACY_TYPE_MAP.put("short", "smallint");
+ LEGACY_TYPE_MAP.put("long", "bigint");
+ LEGACY_TYPE_MAP.put("string", "varchar");
+ LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
+ LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
+ LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
+ LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
+ LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
+ }
+
+ private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
+
+ public static final DataType ANY = DataType.getType("any");
+
+ public static DataType getType(String type) {
+ if (type == null)
+ return null;
+
+ DataType dataType = new DataType(type);
+ DataType cached = CACHE.get(dataType);
+ if (cached == null) {
+ CACHE.put(dataType, dataType);
+ cached = dataType;
+ }
+ return cached;
+ }
+
+ // ============================================================================
+
+ private String name;
+ private int precision;
+ private int scale;
+
+ private DataType(String datatype) {
+ datatype = datatype.trim().toLowerCase();
+ datatype = replaceLegacy(datatype);
+
+ Pattern pattern = TYPE_PATTERN;
+ Matcher m = pattern.matcher(datatype);
+ if (m.matches() == false)
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", does not match " + pattern);
+
+ name = replaceLegacy(m.group(1));
+ precision = -1;
+ scale = -1;
+
+ String leftover = m.group(2);
+ if (leftover != null) {
+ String[] parts = leftover.split("\\s*,\\s*");
+ for (int i = 0; i < parts.length; i++) {
+ int n;
+ try {
+ n = Integer.parseInt(parts[i]);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric");
+ }
+ if (i == 0)
+ precision = n;
+ else if (i == 1)
+ scale = n;
+ else
+ throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts");
+ }
+ }
+
+ // FIXME 256 for unknown string precision
+ if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
+ precision = 256; // to save memory at frontend, e.g. tableau will
+ // allocate memory according to this
+ }
+
+ // FIXME (19,4) for unknown decimal precision
+ if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) {
+ precision = 19;
+ scale = 4;
+ }
+ }
+
+ private String replaceLegacy(String str) {
+ String replace = LEGACY_TYPE_MAP.get(str);
+ return replace == null ? str : replace;
+ }
+
+ public int getStorageBytesEstimate() {
+ return DataTypeSerializer.create(this).getStorageBytesEstimate();
+ }
+
+ public boolean isStringFamily() {
+ return STRING_FAMILY.contains(name);
+ }
+
+ public boolean isIntegerFamily() {
+ return INTEGER_FAMILY.contains(name);
+ }
+
+ public boolean isNumberFamily() {
+ return NUMBER_FAMILY.contains(name);
+ }
+
+ public boolean isDateTimeFamily() {
+ return DATETIME_FAMILY.contains(name);
+ }
+
+ public boolean isDate() {
+ return name.equals("date");
+ }
+
+ public boolean isTime() {
+ return name.equals("time");
+ }
+
+ public boolean isTimestamp() {
+ return name.equals("timestamp");
+ }
+
+ public boolean isDatetime() {
+ return name.equals("datetime");
+ }
+
+ public boolean isTinyInt() {
+ return name.equals("tinyint");
+ }
+
+ public boolean isSmallInt() {
+ return name.equals("smallint");
+ }
+
+ public boolean isInt() {
+ return name.equals("integer");
+ }
+
+ public boolean isBigInt() {
+ return name.equals("bigint");
+ }
+
+ public boolean isFloat() {
+ return name.equals("float");
+ }
+
+ public boolean isDouble() {
+ return name.equals("double");
+ }
+
+ public boolean isDecimal() {
+ return name.equals("decimal");
+ }
+
+ public boolean isHLLC() {
+ return name.equals("hllc");
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + precision;
+ result = prime * result + scale;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ DataType other = (DataType) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (precision != other.precision)
+ return false;
+ if (scale != other.scale)
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ if (precision < 0 && scale < 0)
+ return name;
+ else if (scale < 0)
+ return name + "(" + precision + ")";
+ else
+ return name + "(" + precision + "," + scale + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
new file mode 100644
index 0000000..fd3121f
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.measure.hllc.HLLCSerializer;
+import org.apache.kylin.measure.topn.TopNCounterSerializer;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @author yangli9
+ *
+ * Note: the implementations MUST be thread-safe.
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static Map<String, Class<?>> implementations;
+ static {
+ HashMap<String, Class<?>> impl = Maps.newHashMap();
+ impl.put("varchar", StringSerializer.class);
+ impl.put("decimal", BigDecimalSerializer.class);
+ impl.put("double", DoubleSerializer.class);
+ impl.put("float", DoubleSerializer.class);
+ impl.put("bigint", LongSerializer.class);
+ impl.put("long", LongSerializer.class);
+ impl.put("integer", LongSerializer.class);
+ impl.put("int", LongSerializer.class);
+ impl.put("smallint", LongSerializer.class);
+ impl.put("date", DateTimeSerializer.class);
+ impl.put("datetime", DateTimeSerializer.class);
+ impl.put("timestamp", DateTimeSerializer.class);
+ impl.put("topn", TopNCounterSerializer.class);
+ impl.put("hllc", HLLCSerializer.class);
+ implementations = Collections.unmodifiableMap(impl);
+ }
+
+ public static boolean hasRegistered(String dataTypeName) {
+ return implementations.containsKey(dataTypeName);
+ }
+
+ public static void register(String dataTypeName, Class<? extends DataTypeSerializer<?>> impl) {
+ implementations.put(dataTypeName, impl);
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ return create(DataType.getType(dataType));
+ }
+
+ public static DataTypeSerializer<?> create(DataType type) {
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No DataTypeSerializer for type " + type);
+
+ try {
+ return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** return the max number of bytes to the longest serialization */
+ abstract public int maxLength();
+
+ /** get an estimate of size in bytes of the serialized data */
+ abstract public int getStorageBytesEstimate();
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
new file mode 100644
index 0000000..4f7935c
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
@@ -0,0 +1,49 @@
+package org.apache.kylin.metadata.datatype;
+
+import java.nio.ByteBuffer;
+
+public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public DateTimeSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ out.putLong(value.get());
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(in.getLong());
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleMutable.java
new file mode 100644
index 0000000..5bd75e8
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleMutable.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
+
+ private double v;
+
+ public DoubleMutable() {
+ this(0);
+ }
+
+ public DoubleMutable(double v) {
+ set(v);
+ }
+
+ public double get() {
+ return v;
+ }
+
+ public void set(double v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DoubleMutable)) {
+ return false;
+ }
+ DoubleMutable other = (DoubleMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) Double.doubleToLongBits(v);
+ }
+
+ @Override
+ public int compareTo(DoubleMutable o) {
+ return (v < o.v ? -1 : (v == o.v ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Double.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
new file mode 100644
index 0000000..5accf1d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.nio.ByteBuffer;
+
+/**
+ */
+public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
+
+ public DoubleSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(DoubleMutable value, ByteBuffer out) {
+ out.putDouble(value.get());
+ }
+
+ private DoubleMutable current() {
+ DoubleMutable d = current.get();
+ if (d == null) {
+ d = new DoubleMutable();
+ current.set(d);
+ }
+ return d;
+ }
+
+ @Override
+ public DoubleMutable deserialize(ByteBuffer in) {
+ DoubleMutable d = current();
+ d.set(in.getDouble());
+ return d;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongMutable.java
new file mode 100644
index 0000000..b978049
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongMutable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class LongMutable implements Comparable<LongMutable>, Serializable {
+
+ private long v;
+
+ public LongMutable() {
+ this(0);
+ }
+
+ public LongMutable(long v) {
+ set(v);
+ }
+
+ public long get() {
+ return v;
+ }
+
+ public void set(long v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof LongMutable)) {
+ return false;
+ }
+ LongMutable other = (LongMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) v;
+ }
+
+ @Override
+ public int compareTo(LongMutable o) {
+ long thisValue = this.v;
+ long thatValue = o.v;
+ return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Long.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
new file mode 100644
index 0000000..777f494
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+/**
+ */
+public class LongSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public LongSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ BytesUtil.writeVLong(value.get(), out);
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(BytesUtil.readVLong(in));
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return 9; // vlong: 1 + 8
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 5;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
new file mode 100644
index 0000000..14c8909
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/StringSerializer.java
@@ -0,0 +1,48 @@
+package org.apache.kylin.metadata.datatype;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+public class StringSerializer extends DataTypeSerializer<String> {
+
+ final DataType type;
+ final int maxLength;
+
+ public StringSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 2 byte length, rest is String.toBytes()
+ this.maxLength = 2 + type.getPrecision();
+ }
+
+ @Override
+ public void serialize(String value, ByteBuffer out) {
+ int start = out.position();
+
+ BytesUtil.writeUTFString(value, out);
+
+ if (out.position() - start > maxLength)
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ @Override
+ public String deserialize(ByteBuffer in) {
+ return BytesUtil.readUTFString(in);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return BytesUtil.peekByteArrayLength(in);
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return maxLength;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
index 0e687f7..b894fd3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
@@ -5,8 +5,8 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index f03e736..2ddc75a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index d22d0a3..0c36873 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Collection;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -171,6 +171,14 @@ public class FunctionDesc {
this.parameter = parameter;
}
+ public int getParameterCount() {
+ int count = 0;
+ for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
+ count++;
+ }
+ return count;
+ }
+
public DataType getSQLType() {
if (isCountDistinct() || isTopN())
return DataType.ANY;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 5d65771..c8ac1a9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -18,9 +18,6 @@
package org.apache.kylin.metadata.model;
-import java.util.Collections;
-import java.util.List;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -39,17 +36,6 @@ public class MeasureDesc {
@JsonProperty("dependent_measure_ref")
private String dependentMeasureRef;
- public List<TblColRef> getColumnsNeedDictionary() {
- // measure could store literal values using dictionary encoding to save space, like TopN
- if (function.isTopN()) {
- return Collections.singletonList(function.getTopNLiteralColumn());
- } else {
- return Collections.emptyList();
- }
- }
-
-
-
public String getName() {
return name;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index de27145..61ba73b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index 4a8c5d1..e52bf3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -1,7 +1,7 @@
package org.apache.kylin.metadata.realization;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
new file mode 100644
index 0000000..6d1a7b8
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
@@ -0,0 +1,46 @@
+package org.apache.kylin.aggregation.topn;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.topn.TopNCounterSerializer;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TopNCounterSerializerTest {
+
+ private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getType("topn(10)"));
+
+ @Test
+ public void testSerialization() {
+ TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
+ Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
+ for (Integer i : stream) {
+ vs.offer(new ByteArray(Bytes.toBytes(i)));
+ }
+
+ ByteBuffer out = ByteBuffer.allocate(1024);
+ serializer.serialize(vs, out);
+
+ byte[] copyBytes = new byte[out.position()];
+ System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
+
+ ByteBuffer in = ByteBuffer.wrap(copyBytes);
+ TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
+
+ Assert.assertEquals(vs.toString(), vsNew.toString());
+
+ }
+
+ @Test
+ public void testValueOf() {
+ // FIXME need a good unit test for valueOf()
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
new file mode 100644
index 0000000..f920ee7
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.metadata.datatype;
+
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.metadata.datatype.BigDecimalSerializer;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ */
+public class BigDecimalSerializerTest {
+
+ private static BigDecimalSerializer bigDecimalSerializer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ bigDecimalSerializer = new BigDecimalSerializer(DataType.getType("decimal"));
+ }
+
+ @Test
+ public void testNormal() {
+ BigDecimal input = new BigDecimal("1234.1234");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testScaleOutOfRange() {
+ BigDecimal input = new BigDecimal("1234.1234567890");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutOfPrecision() {
+ BigDecimal input = new BigDecimal("66855344214907231736.4924");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ bigDecimalSerializer.serialize(input, buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
index fbd6f97..0dc1afa 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
@@ -23,8 +23,8 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnOrder;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 4f011cf..3bca687 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -24,11 +24,11 @@ import java.util.List;
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-import org.apache.kylin.common.datatype.DoubleMutable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
index aaf07fe..d32f171 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/translate/ColumnValueRangeTest.java
@@ -6,7 +6,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 3fa7d5c..45cc88e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -6,13 +6,11 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Maps;
-
import org.apache.hadoop.io.Text;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSplitter;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.SplittedBytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -22,11 +20,16 @@ import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +52,8 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
protected String intermediateTableRowDelimiter;
protected byte byteRowDelimiter;
protected int counter;
+ protected MeasureIngester<?>[] aggrIngesters;
+ protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected Object[] measures;
protected byte[][] keyBytesBuf;
protected BytesSplitter bytesSplitter;
@@ -58,7 +63,6 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
private Text outputKey = new Text();
private Text outputValue = new Text();
private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private Map<Integer, Dictionary<String>> topNLiteralColDictMap;
@Override
protected void setup(Context context) throws IOException {
@@ -93,25 +97,12 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
keyBytesBuf = new byte[colCount][];
- initTopNLiteralColDictionaryMap();
+ aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+ dictionaryMap = cubeSegment.buildDictionaryMap();
+
initNullBytes();
}
-
- private void initTopNLiteralColDictionaryMap() {
- topNLiteralColDictMap = Maps.newHashMap();
- for (int measureIdx = 0; measureIdx < measures.length; measureIdx++) {
- MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
- FunctionDesc func = measureDesc.getFunction();
- if (func.isTopN()) {
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
- int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef literalCol = func.getTopNLiteralColumn();
- Dictionary<String> dictionary = (Dictionary<String>) cubeSegment.getDictionary(literalCol);
- topNLiteralColDictMap.put(literalColIdx, dictionary);
- }
- }
- }
-
+
private void initNullBytes() {
nullBytes = Lists.newArrayList();
nullBytes.add(HIVE_NULL);
@@ -146,72 +137,46 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
private void buildValue(SplittedBytes[] splitBuffers) {
for (int i = 0; i < measures.length; i++) {
- byte[] valueBytes = getValueBytes(splitBuffers, i);
- measures[i] = measureCodec.getSerializer(i).valueOf(valueBytes);
+ measures[i] = buildValueOf(i, splitBuffers);
}
valueBuf.clear();
measureCodec.encode(measures, valueBuf);
}
- private byte[] getValueBytes(SplittedBytes[] splitBuffers, int measureIdx) {
- MeasureDesc desc = cubeDesc.getMeasures().get(measureIdx);
- FunctionDesc func = desc.getFunction();
- ParameterDesc paramDesc = func.getParameter();
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
-
- byte[] result = null;
-
- // constant
- if (flatTableIdx == null) {
- result = Bytes.toBytes(paramDesc.getValue());
- }
- // count and count distinct
- else if (func.isCount() || func.isHolisticCountDistinct()) {
- // note for holistic count distinct, this value will be ignored
- result = ONE;
- }
- // topN, need encode the key column
- else if (func.isTopN()) {
- // encode the key column with dict, and get the counter column;
- int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
- Dictionary<String> literalColDict = topNLiteralColDictMap.get(keyColIndex);
- int keyColEncoded = literalColDict.getIdFromValue(Bytes.toString(splitBuffers[keyColIndex].value));
- valueBuf.clear();
- valueBuf.putInt(literalColDict.getSizeOfId());
- valueBuf.putInt(keyColEncoded);
- if (flatTableIdx.length == 1) {
- // only literalCol, use 1.0 as counter
- valueBuf.putDouble(1.0);
+ private Object buildValueOf(int idxOfMeasure, SplittedBytes[] splitBuffers) {
+ MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
+ FunctionDesc function = measure.getFunction();
+ int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
+
+ int paramCount = function.getParameterCount();
+ String[] inputToMeasure = new String[paramCount];
+
+ // pick up parameter values
+ ParameterDesc param = function.getParameter();
+ int colParamIdx = 0; // index among parameters of column type
+ for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
+ String value;
+ if (function.isCount() || function.isHolisticCountDistinct()) {
+ // note for holistic count distinct, this value will be ignored
+ value = "1";
+ } else if (param.isColumnType()) {
+ value = getCell(colIdxOnFlatTable[colParamIdx++], splitBuffers);
} else {
- // get the counter column value
- valueBuf.putDouble(Double.valueOf(Bytes.toString(splitBuffers[flatTableIdx[0]].value)));
- }
-
- result = valueBuf.array();
-
- }
- // normal case, concat column values
- else {
- // for multiple columns, their values are joined
- for (int i = 0; i < flatTableIdx.length; i++) {
- SplittedBytes split = splitBuffers[flatTableIdx[i]];
- if (result == null) {
- result = Arrays.copyOf(split.value, split.length);
- } else {
- byte[] newResult = new byte[result.length + split.length];
- System.arraycopy(result, 0, newResult, 0, result.length);
- System.arraycopy(split.value, 0, newResult, result.length, split.length);
- result = newResult;
- }
+ value = param.getValue();
}
+ inputToMeasure[i] = value;
}
+
+ return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
+ }
- if (isNull(result)) {
- result = null;
- }
-
- return result;
+ private String getCell(int i, SplittedBytes[] splitBuffers) {
+ byte[] bytes = Arrays.copyOf(splitBuffers[i].value, splitBuffers[i].length);
+ if (isNull(bytes))
+ return null;
+ else
+ return Bytes.toString(bytes);
}
protected void outputKV(Context context) throws IOException, InterruptedException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index 0f94dca..381b07c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -23,8 +23,6 @@ import java.nio.ByteBuffer;
import java.util.List;
import org.apache.hadoop.io.Text;
-import org.apache.kylin.aggregation.MeasureAggregators;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
@@ -32,6 +30,8 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index d724c76..752c01d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -16,20 +16,18 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.engine.mr.ByteArrayWritable;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -63,7 +61,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat();
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+ Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
// dictionary
for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index ab87b21..c35e77f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -3,8 +3,6 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.List;
-import org.apache.kylin.aggregation.MeasureAggregators;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -16,6 +14,8 @@ import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 72ce84c..4d37089 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -23,12 +23,12 @@ import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.SplittedBytes;
import org.apache.kylin.cube.CubeInstance;
@@ -40,7 +40,6 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.mr.ByteArrayWritable;
import org.apache.kylin.engine.mr.IMROutput2.IMRStorageInputFormat;
@@ -48,6 +47,7 @@ import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index e49e1a6..c596318 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -27,12 +27,12 @@ import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.SplittedBytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -43,11 +43,11 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
index 8fced61..f247a8f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeDictionaryStep.java
@@ -32,7 +32,6 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.job.exception.ExecuteException;
[04/21] kylin git commit: KYLIN-976 very initial
Posted by li...@apache.org.
KYLIN-976 very initial
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/836f2336
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/836f2336
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/836f2336
Branch: refs/heads/2.x-staging
Commit: 836f2336270a6ec9d7240bb1d436e87630e12f73
Parents: b8ee69c
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Nov 24 13:15:54 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:48:41 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DoubleMutable.java | 68 +++++
.../apache/kylin/common/util/LongMutable.java | 70 +++++
.../apache/kylin/aggregation/Aggregation.java | 42 +++
.../kylin/aggregation/DataTypeSerializer.java | 118 +++++++++
.../kylin/aggregation/MeasureAggregator.java | 133 ++++++++++
.../kylin/aggregation/MeasureAggregators.java | 81 ++++++
.../apache/kylin/aggregation/MeasureCodec.java | 78 ++++++
.../basic/BigDecimalMaxAggregator.java | 54 ++++
.../basic/BigDecimalMinAggregator.java | 55 ++++
.../aggregation/basic/BigDecimalSerializer.java | 111 ++++++++
.../basic/BigDecimalSumAggregator.java | 51 ++++
.../aggregation/basic/DateTimeSerializer.java | 65 +++++
.../aggregation/basic/DoubleMaxAggregator.java | 54 ++++
.../aggregation/basic/DoubleMinAggregator.java | 54 ++++
.../aggregation/basic/DoubleSerializer.java | 84 ++++++
.../aggregation/basic/DoubleSumAggregator.java | 51 ++++
.../aggregation/basic/LongMaxAggregator.java | 54 ++++
.../aggregation/basic/LongMinAggregator.java | 54 ++++
.../kylin/aggregation/basic/LongSerializer.java | 91 +++++++
.../aggregation/basic/LongSumAggregator.java | 51 ++++
.../aggregation/basic/StringSerializer.java | 56 ++++
.../kylin/aggregation/hllc/HLLCAggregator.java | 64 +++++
.../kylin/aggregation/hllc/HLLCSerializer.java | 98 +++++++
.../kylin/aggregation/hllc/LDCAggregator.java | 63 +++++
.../kylin/aggregation/topn/TopNAggregator.java | 66 +++++
.../aggregation/topn/TopNCounterSerializer.java | 117 +++++++++
.../kylin/cube/gridtable/CubeCodeSystem.java | 6 +-
.../cube/gridtable/TrimmedCubeCodeSystem.java | 6 +-
.../cube/inmemcubing/DoggedCubeBuilder.java | 2 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +-
.../InMemCubeBuilderInputConverter.java | 5 +-
.../kylin/gridtable/GTAggregateScanner.java | 2 +-
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +-
.../apache/kylin/gridtable/IGTCodeSystem.java | 2 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 2 +-
.../basic/BigDecimalSerializerTest.java | 53 ++++
.../topn/TopNCounterSerializerTest.java | 61 +++++
.../gridtable/AggregationCacheMemSizeTest.java | 14 +-
.../kylin/gridtable/DictGridTableTest.java | 2 +-
.../kylin/gridtable/SimpleGridTableTest.java | 2 +-
.../gridtable/SimpleInvertedIndexTest.java | 5 +-
.../metadata/measure/MeasureCodecTest.java | 3 +
.../measure/BigDecimalMaxAggregator.java | 52 ----
.../measure/BigDecimalMinAggregator.java | 53 ----
.../measure/BigDecimalSumAggregator.java | 49 ----
.../metadata/measure/DoubleMaxAggregator.java | 51 ----
.../metadata/measure/DoubleMinAggregator.java | 51 ----
.../kylin/metadata/measure/DoubleMutable.java | 68 -----
.../metadata/measure/DoubleSumAggregator.java | 48 ----
.../kylin/metadata/measure/HLLCAggregator.java | 63 -----
.../kylin/metadata/measure/LDCAggregator.java | 60 -----
.../metadata/measure/LongMaxAggregator.java | 51 ----
.../metadata/measure/LongMinAggregator.java | 51 ----
.../kylin/metadata/measure/LongMutable.java | 70 -----
.../metadata/measure/LongSumAggregator.java | 48 ----
.../metadata/measure/MeasureAggregator.java | 121 ---------
.../metadata/measure/MeasureAggregators.java | 81 ------
.../kylin/metadata/measure/MeasureCodec.java | 79 ------
.../kylin/metadata/measure/TopNAggregator.java | 66 -----
.../measure/fixedlen/FixedHLLCodec.java | 80 ------
.../measure/fixedlen/FixedLenMeasureCodec.java | 49 ----
.../measure/fixedlen/FixedPointLongCodec.java | 117 ---------
.../serializer/BigDecimalSerializer.java | 110 --------
.../measure/serializer/DataTypeSerializer.java | 111 --------
.../measure/serializer/DateTimeSerializer.java | 64 -----
.../measure/serializer/DoubleSerializer.java | 83 ------
.../measure/serializer/HLLCSerializer.java | 97 -------
.../measure/serializer/LongSerializer.java | 90 -------
.../measure/serializer/StringSerializer.java | 55 ----
.../serializer/TopNCounterSerializer.java | 116 --------
.../apache/kylin/metadata/model/DataType.java | 2 +-
.../fixedlen/FixedPointLongCodecTest.java | 44 ----
.../serializer/BigDecimalSerializerTest.java | 52 ----
.../serializer/TopNCounterSerializerTest.java | 61 -----
.../org/apache/kylin/storage/tuple/Tuple.java | 4 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 3 +-
.../kylin/engine/mr/steps/CuboidReducer.java | 4 +-
.../engine/mr/steps/InMemCuboidReducer.java | 4 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../kylin/engine/mr/steps/CubeReducerTest.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 8 +-
.../invertedindex/index/RawTableRecord.java | 4 +-
.../kylin/invertedindex/index/TableRecord.java | 2 +-
.../invertedindex/index/TableRecordInfo.java | 2 +-
.../index/TableRecordInfoDigest.java | 4 +-
.../invertedindex/measure/FixedHLLCodec.java | 80 ++++++
.../measure/FixedLenMeasureCodec.java | 49 ++++
.../measure/FixedPointLongCodec.java | 117 +++++++++
.../invertedindex/model/IIKeyValueCodec.java | 2 +-
.../kylin/invertedindex/IIDescManagerTest.java | 104 ++++++++
.../apache/kylin/invertedindex/IIDescTest.java | 67 +++++
.../kylin/invertedindex/IIInstanceTest.java | 74 ++++++
.../invertedindex/InvertedIndexLocalTest.java | 262 +++++++++++++++++++
.../org/apache/kylin/invertedindex/LZFTest.java | 49 ++++
.../invertedindex/IIDescManagerTest.java | 104 --------
.../invertedindex/invertedindex/IIDescTest.java | 67 -----
.../invertedindex/IIInstanceTest.java | 74 ------
.../invertedindex/InvertedIndexLocalTest.java | 262 -------------------
.../invertedindex/invertedindex/LZFTest.java | 49 ----
.../measure/FixedPointLongCodecTest.java | 45 ++++
.../common/coprocessor/AggregationCache.java | 2 +-
.../observer/AggregationScanner.java | 2 +-
.../observer/ObserverAggregationCache.java | 2 +-
.../observer/ObserverAggregators.java | 4 +-
.../endpoint/EndpointAggregationCache.java | 2 +-
.../endpoint/EndpointAggregators.java | 6 +-
.../ii/coprocessor/endpoint/IIEndpoint.java | 3 +-
.../storage/hbase/steps/CubeHFileMapper.java | 2 +-
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../storage/hbase/steps/KeyValueCreator.java | 2 +-
.../storage/hbase/steps/RowValueDecoder.java | 6 +-
.../observer/AggregateRegionObserverTest.java | 2 +-
.../endpoint/EndpointAggregationTest.java | 7 +-
.../hbase/steps/CubeHFileMapper2Test.java | 2 +-
.../hbase/steps/RowValueDecoderTest.java | 4 +-
116 files changed, 2924 insertions(+), 2819 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java b/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
new file mode 100644
index 0000000..520cd74
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DoubleMutable.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
+
+ private double v;
+
+ public DoubleMutable() {
+ this(0);
+ }
+
+ public DoubleMutable(double v) {
+ set(v);
+ }
+
+ public double get() {
+ return v;
+ }
+
+ public void set(double v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof DoubleMutable)) {
+ return false;
+ }
+ DoubleMutable other = (DoubleMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) Double.doubleToLongBits(v);
+ }
+
+ @Override
+ public int compareTo(DoubleMutable o) {
+ return (v < o.v ? -1 : (v == o.v ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Double.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java b/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
new file mode 100644
index 0000000..238bb86
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/LongMutable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.Serializable;
+
+@SuppressWarnings("serial")
+public class LongMutable implements Comparable<LongMutable>, Serializable {
+
+ private long v;
+
+ public LongMutable() {
+ this(0);
+ }
+
+ public LongMutable(long v) {
+ set(v);
+ }
+
+ public long get() {
+ return v;
+ }
+
+ public void set(long v) {
+ this.v = v;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof LongMutable)) {
+ return false;
+ }
+ LongMutable other = (LongMutable) o;
+ return this.v == other.v;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) v;
+ }
+
+ @Override
+ public int compareTo(LongMutable o) {
+ long thisValue = this.v;
+ long thatValue = o.v;
+ return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
+ }
+
+ @Override
+ public String toString() {
+ return Long.toString(v);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
new file mode 100644
index 0000000..193c5de
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/Aggregation.java
@@ -0,0 +1,42 @@
+package org.apache.kylin.aggregation;
+
+import java.util.List;
+
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+abstract public class Aggregation {
+
+ /* ============================================================================
+ * Define
+ * ---------------------------------------------------------------------------- */
+
+ abstract public DataType getAggregationDataType();
+
+ abstract public DataType getResultDataType();
+
+ abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
+
+ /* ============================================================================
+ * Build
+ * ---------------------------------------------------------------------------- */
+
+ abstract public DataTypeSerializer<?> getSeralizer();
+
+ abstract public MeasureAggregator<?> getAggregator();
+
+ abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
+
+ abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
+
+ /* ============================================================================
+ * Cube Selection
+ * ---------------------------------------------------------------------------- */
+
+ /* ============================================================================
+ * Query
+ * ---------------------------------------------------------------------------- */
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
new file mode 100644
index 0000000..df6833c
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/DataTypeSerializer.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.aggregation.basic.BigDecimalSerializer;
+import org.apache.kylin.aggregation.basic.DateTimeSerializer;
+import org.apache.kylin.aggregation.basic.DoubleSerializer;
+import org.apache.kylin.aggregation.basic.LongSerializer;
+import org.apache.kylin.aggregation.basic.StringSerializer;
+import org.apache.kylin.aggregation.hllc.HLLCSerializer;
+import org.apache.kylin.aggregation.topn.TopNCounterSerializer;
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.metadata.model.DataType;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @author yangli9
+ *
+ * Note: the implementations MUST be thread-safe.
+ *
+ */
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+
+ final static Map<String, Class<?>> implementations;
+ static {
+ HashMap<String, Class<?>> impl = Maps.newHashMap();
+ impl.put("varchar", StringSerializer.class);
+ impl.put("decimal", BigDecimalSerializer.class);
+ impl.put("double", DoubleSerializer.class);
+ impl.put("float", DoubleSerializer.class);
+ impl.put("bigint", LongSerializer.class);
+ impl.put("long", LongSerializer.class);
+ impl.put("integer", LongSerializer.class);
+ impl.put("int", LongSerializer.class);
+ impl.put("smallint", LongSerializer.class);
+ impl.put("date", DateTimeSerializer.class);
+ impl.put("datetime", DateTimeSerializer.class);
+ impl.put("timestamp", DateTimeSerializer.class);
+ implementations = Collections.unmodifiableMap(impl);
+
+ }
+
+ public static DataTypeSerializer<?> create(String dataType) {
+ return create(DataType.getInstance(dataType));
+ }
+
+ public static DataTypeSerializer<?> create(DataType type) {
+ if (type.isHLLC()) {
+ return new HLLCSerializer(type);
+ }
+
+ if (type.isTopN()) {
+ return new TopNCounterSerializer(type);
+ }
+
+ Class<?> clz = implementations.get(type.getName());
+ if (clz == null)
+ throw new RuntimeException("No MeasureSerializer for type " + type);
+
+ try {
+ return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
+ } catch (Exception e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** peek into buffer and return the length of serialization */
+ abstract public int peekLength(ByteBuffer in);
+
+ /** return the max number of bytes to the longest serialization */
+ abstract public int maxLength();
+
+ /** get an estimate of size in bytes of the serialized data */
+ abstract public int getStorageBytesEstimate();
+
+ /** convert from String to obj (string often come as byte[] in mapred) */
+ abstract public T valueOf(byte[] value);
+
+ /** convert from String to obj */
+ public T valueOf(String value) {
+ try {
+ return valueOf(value.getBytes("UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e); // never happen
+ }
+ }
+
+ /** convert from obj to string */
+ public String toString(T value) {
+ if (value == null)
+ return "NULL";
+ else
+ return value.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
new file mode 100644
index 0000000..9c8945d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation;
+
+import org.apache.kylin.aggregation.basic.BigDecimalMaxAggregator;
+import org.apache.kylin.aggregation.basic.BigDecimalMinAggregator;
+import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
+import org.apache.kylin.aggregation.basic.DoubleMaxAggregator;
+import org.apache.kylin.aggregation.basic.DoubleMinAggregator;
+import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
+import org.apache.kylin.aggregation.basic.LongMaxAggregator;
+import org.apache.kylin.aggregation.basic.LongMinAggregator;
+import org.apache.kylin.aggregation.basic.LongSumAggregator;
+import org.apache.kylin.aggregation.hllc.HLLCAggregator;
+import org.apache.kylin.aggregation.hllc.LDCAggregator;
+import org.apache.kylin.aggregation.topn.TopNAggregator;
+import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+import java.io.Serializable;
+
+/**
+ */
+@SuppressWarnings("serial")
+abstract public class MeasureAggregator<V> implements Serializable {
+
+ public static MeasureAggregator<?> create(String funcName, String returnType) {
+ if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) {
+ if (isInteger(returnType))
+ return new LongSumAggregator();
+ else if (isBigDecimal(returnType))
+ return new BigDecimalSumAggregator();
+ else if (isDouble(returnType))
+ return new DoubleSumAggregator();
+ } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
+ DataType hllcType = DataType.getInstance(returnType);
+ if (hllcType.isHLLC())
+ return new HLLCAggregator(hllcType.getPrecision());
+ else
+ return new LDCAggregator();
+ } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
+ if (isInteger(returnType))
+ return new LongMaxAggregator();
+ else if (isBigDecimal(returnType))
+ return new BigDecimalMaxAggregator();
+ else if (isDouble(returnType))
+ return new DoubleMaxAggregator();
+ } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) {
+ if (isInteger(returnType))
+ return new LongMinAggregator();
+ else if (isBigDecimal(returnType))
+ return new BigDecimalMinAggregator();
+ else if (isDouble(returnType))
+ return new DoubleMinAggregator();
+ } else if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName)) {
+ return new TopNAggregator();
+ }
+ throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'");
+ }
+
+ public static boolean isBigDecimal(String type) {
+ return type.startsWith("decimal");
+ }
+
+ public static boolean isDouble(String type) {
+ return "double".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "real".equalsIgnoreCase(type);
+ }
+
+ public static boolean isInteger(String type) {
+ return "long".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type) || "integer".equalsIgnoreCase(type);
+ }
+
+ public static int guessBigDecimalMemBytes() {
+ // 116 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 8 // ref to BigDecimal
+ + 8 // BigDecimal obj shell
+ + 100; // guess of BigDecimal internal
+ }
+
+ public static int guessDoubleMemBytes() {
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to DoubleWritable
+ + 8 // DoubleWritable obj shell
+ + 8; // size of double
+ */
+ }
+
+ public static int guessLongMemBytes() {
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to LongWritable
+ + 8 // LongWritable obj shell
+ + 8; // size of long
+ */
+ }
+
+ // ============================================================================
+
+ @SuppressWarnings("rawtypes")
+ public void setDependentAggregator(MeasureAggregator agg) {
+ }
+
+ abstract public void reset();
+
+ abstract public void aggregate(V value);
+
+ abstract public V getState();
+
+ // get an estimate of memory consumption UPPER BOUND
+ abstract public int getMemBytesEstimate();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
new file mode 100644
index 0000000..3aa575b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+/**
+ */
+@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
+public class MeasureAggregators implements Serializable {
+
+ private final MeasureAggregator[] aggs;
+ private final int descLength;
+
+ public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
+ this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
+ }
+
+ public MeasureAggregators(MeasureDesc... measureDescs) {
+ descLength = measureDescs.length;
+ aggs = new MeasureAggregator[descLength];
+
+ Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
+ for (int i = 0; i < descLength; i++) {
+ FunctionDesc func = measureDescs[i].getFunction();
+ aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
+ measureIndexMap.put(measureDescs[i].getName(), i);
+ }
+ // fill back dependent aggregator
+ for (int i = 0; i < descLength; i++) {
+ String depMsrRef = measureDescs[i].getDependentMeasureRef();
+ if (depMsrRef != null) {
+ int index = measureIndexMap.get(depMsrRef);
+ aggs[i].setDependentAggregator(aggs[index]);
+ }
+ }
+ }
+
+ public void reset() {
+ for (int i = 0; i < aggs.length; i++) {
+ aggs[i].reset();
+ }
+ }
+
+ public void aggregate(Object[] values) {
+ assert values.length == descLength;
+
+ for (int i = 0; i < descLength; i++) {
+ aggs[i].aggregate(values[i]);
+ }
+ }
+
+ public void collectStates(Object[] states) {
+ for (int i = 0; i < descLength; i++) {
+ states[i] = aggs[i].getState();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
new file mode 100644
index 0000000..cbcb3a8
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class MeasureCodec {
+
+ int nMeasures;
+ DataTypeSerializer[] serializers;
+
+ public MeasureCodec(Collection<MeasureDesc> measureDescs) {
+ this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
+ }
+
+ public MeasureCodec(MeasureDesc... measureDescs) {
+ String[] dataTypes = new String[measureDescs.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ dataTypes[i] = measureDescs[i].getFunction().getReturnType();
+ }
+ init(dataTypes);
+ }
+
+ public MeasureCodec(String... dataTypes) {
+ init(dataTypes);
+ }
+
+ private void init(String[] dataTypes) {
+ nMeasures = dataTypes.length;
+ serializers = new DataTypeSerializer[nMeasures];
+
+ for (int i = 0; i < nMeasures; i++) {
+ serializers[i] = DataTypeSerializer.create(dataTypes[i]);
+ }
+ }
+
+ public DataTypeSerializer getSerializer(int idx) {
+ return serializers[idx];
+ }
+
+ public void decode(ByteBuffer buf, Object[] result) {
+ assert result.length == nMeasures;
+ for (int i = 0; i < nMeasures; i++) {
+ result[i] = serializers[i].deserialize(buf);
+ }
+ }
+
+ public void encode(Object[] values, ByteBuffer out) {
+ assert values.length == nMeasures;
+ for (int i = 0; i < nMeasures; i++) {
+ serializers[i].serialize(values[i], out);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
new file mode 100644
index 0000000..ca044d0
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ if (max == null)
+ max = value;
+ else if (max.compareTo(value) < 0)
+ max = value;
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
new file mode 100644
index 0000000..3c3c85e
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ if (max == null)
+ max = value;
+ else if (max.compareTo(value) > 0)
+ max = value;
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
new file mode 100644
index 0000000..9f7c3cf
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSerializer.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.model.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author yangli9
+ *
+ */
+public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
+
+ private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
+
+ final DataType type;
+ final int maxLength;
+
+ int avoidVerbose = 0;
+
+ public BigDecimalSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
+ this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
+ }
+
+ @Override
+ public void serialize(BigDecimal value, ByteBuffer out) {
+ if (value.scale() > type.getScale()) {
+ if (avoidVerbose % 10000 == 0) {
+ logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
+ }
+ value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
+ }
+ byte[] bytes = value.unscaledValue().toByteArray();
+ if (bytes.length + 2 > maxLength) {
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ BytesUtil.writeVInt(value.scale(), out);
+ BytesUtil.writeVInt(bytes.length, out);
+ out.put(bytes);
+ }
+
+ @Override
+ public BigDecimal deserialize(ByteBuffer in) {
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+
+ byte[] bytes = new byte[n];
+ in.get(bytes);
+
+ return new BigDecimal(new BigInteger(bytes), scale);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ @SuppressWarnings("unused")
+ int scale = BytesUtil.readVInt(in);
+ int n = BytesUtil.readVInt(in);
+ int len = in.position() - mark + n;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public BigDecimal valueOf(byte[] value) {
+ if (value == null)
+ return new BigDecimal(0);
+ else
+ return new BigDecimal(Bytes.toString(value));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
new file mode 100644
index 0000000..19aef3c
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal sum = new BigDecimal(0);
+
+ @Override
+ public void reset() {
+ sum = new BigDecimal(0);
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ sum = sum.add(value);
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
new file mode 100644
index 0000000..0bf4aba
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DateTimeSerializer.java
@@ -0,0 +1,65 @@
+package org.apache.kylin.aggregation.basic;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.metadata.model.DataType;
+
+public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public DateTimeSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ out.putLong(value.get());
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(in.getLong());
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public LongMutable valueOf(byte[] value) {
+ LongMutable l = current();
+ if (value == null)
+ l.set(0L);
+ else
+ l.set(DateFormat.stringToMillis(Bytes.toString(value)));
+ return l;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
new file mode 100644
index 0000000..f09614d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ if (max == null)
+ max = new DoubleMutable(value.get());
+ else if (max.get() < value.get())
+ max.set(value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
new file mode 100644
index 0000000..b93c15c
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable min = null;
+
+ @Override
+ public void reset() {
+ min = null;
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ if (min == null)
+ min = new DoubleMutable(value.get());
+ else if (min.get() > value.get())
+ min.set(value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return min;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
new file mode 100644
index 0000000..f207054
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSerializer.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ */
+public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
+
+ public DoubleSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(DoubleMutable value, ByteBuffer out) {
+ out.putDouble(value.get());
+ }
+
+ private DoubleMutable current() {
+ DoubleMutable d = current.get();
+ if (d == null) {
+ d = new DoubleMutable();
+ current.set(d);
+ }
+ return d;
+ }
+
+ @Override
+ public DoubleMutable deserialize(ByteBuffer in) {
+ DoubleMutable d = current();
+ d.set(in.getDouble());
+ return d;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 8;
+ }
+
+ @Override
+ public int maxLength() {
+ return 8;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 8;
+ }
+
+ @Override
+ public DoubleMutable valueOf(byte[] value) {
+ DoubleMutable d = current();
+ if (value == null)
+ d.set(0d);
+ else
+ d.set(Double.parseDouble(Bytes.toString(value)));
+ return d;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
new file mode 100644
index 0000000..298cec6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable sum = new DoubleMutable();
+
+ @Override
+ public void reset() {
+ sum.set(0.0);
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ sum.set(sum.get() + value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
new file mode 100644
index 0000000..71d95f2
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ if (max == null)
+ max = new LongMutable(value.get());
+ else if (max.get() < value.get())
+ max.set(value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
new file mode 100644
index 0000000..d1e93f2
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongMinAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable min = null;
+
+ @Override
+ public void reset() {
+ min = null;
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ if (min == null)
+ min = new LongMutable(value.get());
+ else if (min.get() > value.get())
+ min.set(value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return min;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
new file mode 100644
index 0000000..202596d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ */
+public class LongSerializer extends DataTypeSerializer<LongMutable> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+
+ public LongSerializer(DataType type) {
+ }
+
+ @Override
+ public void serialize(LongMutable value, ByteBuffer out) {
+ BytesUtil.writeVLong(value.get(), out);
+ }
+
+ private LongMutable current() {
+ LongMutable l = current.get();
+ if (l == null) {
+ l = new LongMutable();
+ current.set(l);
+ }
+ return l;
+ }
+
+ @Override
+ public LongMutable deserialize(ByteBuffer in) {
+ LongMutable l = current();
+ l.set(BytesUtil.readVLong(in));
+ return l;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+
+ BytesUtil.readVLong(in);
+ int len = in.position() - mark;
+
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return 9; // vlong: 1 + 8
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 5;
+ }
+
+ @Override
+ public LongMutable valueOf(byte[] value) {
+ LongMutable l = current();
+ if (value == null)
+ l.set(0L);
+ else
+ l.set(Long.parseLong(Bytes.toString(value)));
+ return l;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
new file mode 100644
index 0000000..c85c83c
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.basic;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongSumAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable sum = new LongMutable();
+
+ @Override
+ public void reset() {
+ sum.set(0);
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ sum.set(sum.get() + value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
new file mode 100644
index 0000000..e84278d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
@@ -0,0 +1,56 @@
+package org.apache.kylin.aggregation.basic;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.model.DataType;
+
+public class StringSerializer extends DataTypeSerializer<String> {
+
+ final DataType type;
+ final int maxLength;
+
+ public StringSerializer(DataType type) {
+ this.type = type;
+ // see serialize(): 2 byte length, rest is String.toBytes()
+ this.maxLength = 2 + type.getPrecision();
+ }
+
+ @Override
+ public void serialize(String value, ByteBuffer out) {
+ int start = out.position();
+
+ BytesUtil.writeUTFString(value, out);
+
+ if (out.position() - start > maxLength)
+ throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
+ }
+
+ @Override
+ public String deserialize(ByteBuffer in) {
+ return BytesUtil.readUTFString(in);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return BytesUtil.peekByteArrayLength(in);
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return maxLength;
+ }
+
+ @Override
+ public String valueOf(byte[] value) {
+ return Bytes.toString(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
new file mode 100644
index 0000000..8f85fe8
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.hllc;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
+
+ final int precision;
+ HyperLogLogPlusCounter sum = null;
+
+ public HLLCAggregator(int precision) {
+ this.precision = precision;
+ }
+
+ @Override
+ public void reset() {
+ sum = null;
+ }
+
+ @Override
+ public void aggregate(HyperLogLogPlusCounter value) {
+ if (sum == null)
+ sum = new HyperLogLogPlusCounter(value);
+ else
+ sum.merge(value);
+ }
+
+ @Override
+ public HyperLogLogPlusCounter getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ // 1024 + 60 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 4 // precision
+ + 8 // ref to HLLC
+ + 8 // HLLC obj shell
+ + 32 + (1 << precision); // HLLC internal
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
new file mode 100644
index 0000000..f7804f4
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.hllc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
+
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
+
+ private int precision;
+
+ public HLLCSerializer(DataType type) {
+ this.precision = type.getPrecision();
+ }
+
+ @Override
+ public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
+ try {
+ value.writeRegisters(out);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private HyperLogLogPlusCounter current() {
+ HyperLogLogPlusCounter hllc = current.get();
+ if (hllc == null) {
+ hllc = new HyperLogLogPlusCounter(precision);
+ current.set(hllc);
+ }
+ return hllc;
+ }
+
+ @Override
+ public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
+ HyperLogLogPlusCounter hllc = current();
+ try {
+ hllc.readRegisters(in);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return hllc;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return current().peekLength(in);
+ }
+
+ @Override
+ public int maxLength() {
+ return current().maxLength();
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return current().maxLength();
+ }
+
+ @Override
+ public HyperLogLogPlusCounter valueOf(byte[] value) {
+ HyperLogLogPlusCounter hllc = current();
+ hllc.clear();
+ if (value == null)
+ hllc.add("__nUlL__");
+ else
+ hllc.add(value);
+ return hllc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
new file mode 100644
index 0000000..643bcae
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.hllc;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.util.LongMutable;
+
+/**
+ * Long Distinct Count
+ */
+@SuppressWarnings("serial")
+public class LDCAggregator extends MeasureAggregator<LongMutable> {
+
+ private static LongMutable ZERO = new LongMutable(0);
+
+ private HLLCAggregator hllAgg = null;
+ private LongMutable state = new LongMutable(0);
+
+ @SuppressWarnings("rawtypes")
+ public void setDependentAggregator(MeasureAggregator agg) {
+ this.hllAgg = (HLLCAggregator) agg;
+ }
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ }
+
+ @Override
+ public LongMutable getState() {
+ if (hllAgg == null) {
+ return ZERO;
+ } else {
+ state.set(hllAgg.getState().getCountEstimate());
+ return state;
+ }
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
new file mode 100644
index 0000000..4f6c7ee
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.topn;
+
+import java.util.Map;
+
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+
+import com.google.common.collect.Maps;
+
+/**
+ *
+ */
+@SuppressWarnings("serial")
+public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
+
+ int capacity = 0;
+ TopNCounter<ByteArray> sum = null;
+ Map<ByteArray, Double> sanityCheckMap;
+
+ @Override
+ public void reset() {
+ sum = null;
+ }
+
+ @Override
+ public void aggregate(TopNCounter<ByteArray> value) {
+ if (sum == null) {
+ capacity = value.getCapacity();
+ sum = new TopNCounter<>(capacity);
+ sanityCheckMap = Maps.newHashMap();
+ }
+ sum.merge(value);
+ }
+
+ @Override
+ public TopNCounter<ByteArray> getState() {
+
+ //sum.retain(capacity);
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return 8 * capacity / 4;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
new file mode 100644
index 0000000..8c44f8f
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.topn;
+
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.topn.Counter;
+import org.apache.kylin.common.topn.DoubleDeltaSerializer;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.model.DataType;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ *
+ */
+public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteArray>> {
+
+ private DoubleDeltaSerializer dds = new DoubleDeltaSerializer(3);
+
+ private int precision;
+
+ public TopNCounterSerializer(DataType dataType) {
+ this.precision = dataType.getPrecision();
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ @SuppressWarnings("unused")
+ int capacity = in.getInt();
+ int size = in.getInt();
+ int keyLength = in.getInt();
+ dds.deserialize(in);
+ int len = in.position() - mark + keyLength * size;
+ in.position(mark);
+ return len;
+ }
+
+ @Override
+ public int maxLength() {
+ return precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8);
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return precision * TopNCounter.EXTRA_SPACE_RATE * 8;
+ }
+
+ @Override
+ public TopNCounter<ByteArray> valueOf(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ int sizeOfId = buffer.getInt();
+ int keyEncodedValue = buffer.getInt();
+ double counter = buffer.getDouble();
+
+ ByteArray key = new ByteArray(sizeOfId);
+ BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, sizeOfId);
+
+ TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(precision * TopNCounter.EXTRA_SPACE_RATE);
+ topNCounter.offer(key, counter);
+ return topNCounter;
+ }
+
+ @Override
+ public void serialize(TopNCounter<ByteArray> value, ByteBuffer out) {
+ double[] counters = value.getCounters();
+ List<ByteArray> peek = value.peek(1);
+ int keyLength = peek.size() > 0 ? peek.get(0).length() : 0;
+ out.putInt(value.getCapacity());
+ out.putInt(value.size());
+ out.putInt(keyLength);
+ dds.serialize(counters, out);
+ Iterator<Counter<ByteArray>> iterator = value.iterator();
+ while (iterator.hasNext()) {
+ out.put(iterator.next().getItem().array());
+ }
+ }
+
+ @Override
+ public TopNCounter<ByteArray> deserialize(ByteBuffer in) {
+ int capacity = in.getInt();
+ int size = in.getInt();
+ int keyLength = in.getInt();
+ double[] counters = dds.deserialize(in);
+
+ TopNCounter<ByteArray> counter = new TopNCounter<ByteArray>(capacity);
+ ByteArray byteArray;
+ for (int i = 0; i < size; i++) {
+ byteArray = new ByteArray(keyLength);
+ in.get(byteArray.array());
+ counter.offerToHead(byteArray, counters[i]);
+ }
+
+ return counter;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index 65d639f..b88f9df 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -6,6 +6,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.basic.StringSerializer;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -15,9 +18,6 @@ import org.apache.kylin.gridtable.DefaultGTComparator;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
-import org.apache.kylin.metadata.measure.serializer.StringSerializer;
/**
* defines how column values will be encoded to/ decoded from GTRecord
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index e4f32fb..26f1636 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -23,13 +23,13 @@ package org.apache.kylin.cube.gridtable;
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.basic.StringSerializer;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
-import org.apache.kylin.metadata.measure.serializer.StringSerializer;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class TrimmedCubeCodeSystem implements IGTCodeSystem {
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 7fe2122..ce912a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -29,6 +29,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.TimeUnit;
+import org.apache.kylin.aggregation.MeasureAggregators;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
@@ -37,7 +38,6 @@ import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.metadata.measure.MeasureAggregators;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index e9d940a..8c6146b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.DoubleMutable;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -46,7 +47,6 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.metadata.measure.DoubleMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index 69a9fc9..951c054 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -18,15 +18,16 @@
package org.apache.kylin.cube.inmemcubing;
import com.google.common.base.Preconditions;
+
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index cd2881e..01696e8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -7,10 +7,10 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[12/21] kylin git commit: KYLIN-976 Add ingester;
Build part done, in-mem cube test pass
Posted by li...@apache.org.
KYLIN-976 Add ingester; Build part done, in-mem cube test pass
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/99396828
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/99396828
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/99396828
Branch: refs/heads/2.x-staging
Commit: 9939682864a91c5b94a89856d35892d692b6e322
Parents: b2d82d3
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Nov 26 18:20:48 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:52:09 2015 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 2 +-
.../streaming/StreamingTableDataGenerator.java | 2 +-
.../common/datatype/BigDecimalSerializer.java | 109 -------
.../apache/kylin/common/datatype/DataType.java | 289 -------------------
.../common/datatype/DataTypeSerializer.java | 108 -------
.../common/datatype/DateTimeSerializer.java | 62 ----
.../kylin/common/datatype/DoubleMutable.java | 68 -----
.../kylin/common/datatype/DoubleSerializer.java | 81 ------
.../kylin/common/datatype/LongMutable.java | 70 -----
.../kylin/common/datatype/LongSerializer.java | 88 ------
.../kylin/common/datatype/StringSerializer.java | 54 ----
.../apache/kylin/common/util/Dictionary.java | 232 +++++++++++++++
.../datatype/BigDecimalSerializerTest.java | 51 ----
.../kylin/aggregation/AggregationType.java | 97 -------
.../kylin/aggregation/IAggregationFactory.java | 6 -
.../kylin/aggregation/MeasureAggregator.java | 76 -----
.../kylin/aggregation/MeasureAggregators.java | 81 ------
.../apache/kylin/aggregation/MeasureCodec.java | 79 -----
.../aggregation/basic/BasicAggregation.java | 128 --------
.../basic/BasicAggregationFactory.java | 31 --
.../basic/BigDecimalMaxAggregator.java | 54 ----
.../basic/BigDecimalMinAggregator.java | 55 ----
.../basic/BigDecimalSumAggregator.java | 51 ----
.../aggregation/basic/DoubleMaxAggregator.java | 54 ----
.../aggregation/basic/DoubleMinAggregator.java | 54 ----
.../aggregation/basic/DoubleSumAggregator.java | 51 ----
.../aggregation/basic/LongMaxAggregator.java | 54 ----
.../aggregation/basic/LongMinAggregator.java | 54 ----
.../aggregation/basic/LongSumAggregator.java | 51 ----
.../kylin/aggregation/hllc/HLLCAggregation.java | 78 -----
.../hllc/HLLCAggregationFactory.java | 35 ---
.../kylin/aggregation/hllc/HLLCAggregator.java | 64 ----
.../kylin/aggregation/hllc/HLLCSerializer.java | 98 -------
.../kylin/aggregation/hllc/LDCAggregator.java | 63 ----
.../kylin/aggregation/topn/TopNAggregation.java | 76 -----
.../topn/TopNAggregationFactory.java | 35 ---
.../kylin/aggregation/topn/TopNAggregator.java | 66 -----
.../aggregation/topn/TopNCounterSerializer.java | 117 --------
.../java/org/apache/kylin/cube/CubeManager.java | 7 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 13 +-
.../kylin/cube/gridtable/CubeCodeSystem.java | 30 +-
.../kylin/cube/gridtable/CubeGridTable.java | 13 +-
.../gridtable/CuboidToGridTableMapping.java | 2 +-
.../gridtable/NotEnoughGTInfoException.java | 1 +
.../cube/gridtable/TrimmedCubeCodeSystem.java | 4 +-
.../inmemcubing/AbstractInMemCubeBuilder.java | 6 +-
.../cube/inmemcubing/DoggedCubeBuilder.java | 6 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 18 +-
.../InMemCubeBuilderInputConverter.java | 105 +++----
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 3 +-
.../kylin/cube/kv/AbstractRowKeyEncoder.java | 2 +-
.../apache/kylin/cube/kv/RowKeyColumnIO.java | 2 +-
.../apache/kylin/cube/kv/RowKeyColumnOrder.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 4 +-
.../model/validation/rule/FunctionRule.java | 2 +-
.../org/apache/kylin/cube/util/CubingUtils.java | 19 +-
.../kylin/gridtable/DefaultGTComparator.java | 2 +-
.../kylin/gridtable/GTAggregateScanner.java | 2 +-
.../java/org/apache/kylin/gridtable/GTInfo.java | 2 +-
.../kylin/gridtable/GTSampleCodeSystem.java | 4 +-
.../apache/kylin/gridtable/IGTCodeSystem.java | 2 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 4 +-
.../topn/TopNCounterSerializerTest.java | 60 ----
.../kylin/cube/DictionaryManagerTest.java | 2 +-
.../DoggedCubeBuilderStressTest.java | 4 +-
.../cube/inmemcubing/DoggedCubeBuilderTest.java | 4 +-
.../cube/inmemcubing/InMemCubeBuilderTest.java | 12 +-
.../gridtable/AggregationCacheMemSizeTest.java | 14 +-
.../kylin/gridtable/DictGridTableTest.java | 6 +-
.../kylin/gridtable/SimpleGridTableTest.java | 2 +-
.../gridtable/SimpleInvertedIndexTest.java | 6 +-
.../metadata/measure/MeasureCodecTest.java | 6 +-
.../apache/kylin/dict/DateStrDictionary.java | 1 +
.../org/apache/kylin/dict/DictCodeSystem.java | 1 +
.../java/org/apache/kylin/dict/Dictionary.java | 233 ---------------
.../apache/kylin/dict/DictionaryGenerator.java | 9 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 1 +
.../kylin/dict/DictionaryInfoSerializer.java | 1 +
.../apache/kylin/dict/DictionaryManager.java | 3 +-
.../apache/kylin/dict/DictionarySerializer.java | 1 +
.../org/apache/kylin/dict/IDictionaryAware.java | 1 +
.../dict/MultipleDictionaryValueEnumerator.java | 2 +
.../apache/kylin/dict/TimeStrDictionary.java | 1 +
.../org/apache/kylin/dict/TrieDictionary.java | 2 +
.../apache/kylin/dict/lookup/SnapshotTable.java | 3 +-
.../apache/kylin/dict/NumberDictionaryTest.java | 3 +-
.../apache/kylin/measure/IMeasureFactory.java | 24 ++
.../apache/kylin/measure/MeasureAggregator.java | 76 +++++
.../kylin/measure/MeasureAggregators.java | 81 ++++++
.../org/apache/kylin/measure/MeasureCodec.java | 79 +++++
.../apache/kylin/measure/MeasureIngester.java | 44 +++
.../org/apache/kylin/measure/MeasureType.java | 103 +++++++
.../measure/basic/BasicMeasureFactory.java | 32 ++
.../kylin/measure/basic/BasicMeasureType.java | 135 +++++++++
.../kylin/measure/basic/BigDecimalIngester.java | 40 +++
.../measure/basic/BigDecimalMaxAggregator.java | 54 ++++
.../measure/basic/BigDecimalMinAggregator.java | 55 ++++
.../measure/basic/BigDecimalSumAggregator.java | 51 ++++
.../kylin/measure/basic/DoubleIngester.java | 45 +++
.../measure/basic/DoubleMaxAggregator.java | 54 ++++
.../measure/basic/DoubleMinAggregator.java | 54 ++++
.../measure/basic/DoubleSumAggregator.java | 51 ++++
.../kylin/measure/basic/LongIngester.java | 45 +++
.../kylin/measure/basic/LongMaxAggregator.java | 54 ++++
.../kylin/measure/basic/LongMinAggregator.java | 54 ++++
.../kylin/measure/basic/LongSumAggregator.java | 51 ++++
.../measure/hllc/HLLCAggregationFactory.java | 36 +++
.../kylin/measure/hllc/HLLCAggregator.java | 64 ++++
.../kylin/measure/hllc/HLLCMeasureType.java | 100 +++++++
.../kylin/measure/hllc/HLLCSerializer.java | 87 ++++++
.../kylin/measure/hllc/LDCAggregator.java | 63 ++++
.../kylin/measure/topn/TopNAggregator.java | 66 +++++
.../measure/topn/TopNCounterSerializer.java | 101 +++++++
.../kylin/measure/topn/TopNMeasureFactory.java | 36 +++
.../kylin/measure/topn/TopNMeasureType.java | 111 +++++++
.../metadata/datatype/BigDecimalSerializer.java | 100 +++++++
.../kylin/metadata/datatype/DataType.java | 278 ++++++++++++++++++
.../metadata/datatype/DataTypeSerializer.java | 99 +++++++
.../metadata/datatype/DateTimeSerializer.java | 49 ++++
.../kylin/metadata/datatype/DoubleMutable.java | 68 +++++
.../metadata/datatype/DoubleSerializer.java | 69 +++++
.../kylin/metadata/datatype/LongMutable.java | 70 +++++
.../kylin/metadata/datatype/LongSerializer.java | 77 +++++
.../metadata/datatype/StringSerializer.java | 48 +++
.../filter/TimeConditionLiteralsReplacer.java | 2 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 2 +-
.../kylin/metadata/model/FunctionDesc.java | 10 +-
.../kylin/metadata/model/MeasureDesc.java | 14 -
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../metadata/realization/SQLDigestUtil.java | 2 +-
.../topn/TopNCounterSerializerTest.java | 46 +++
.../datatype/BigDecimalSerializerTest.java | 53 ++++
.../storage/translate/ColumnValueRange.java | 2 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 4 +-
.../storage/translate/ColumnValueRangeTest.java | 2 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 123 +++-----
.../kylin/engine/mr/steps/CuboidReducer.java | 4 +-
.../engine/mr/steps/InMemCuboidMapper.java | 6 +-
.../engine/mr/steps/InMemCuboidReducer.java | 4 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 4 +-
.../engine/mr/steps/MergeCuboidMapper.java | 4 +-
.../engine/mr/steps/MergeDictionaryStep.java | 1 -
.../kylin/engine/mr/steps/CubeReducerTest.java | 4 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 12 +-
.../streaming/OneOffStreamingBuilder.java | 4 +-
.../engine/streaming/StreamingBatchBuilder.java | 6 +-
.../streaming/cube/StreamingCubeBuilder.java | 10 +-
.../apache/kylin/invertedindex/IISegment.java | 3 +-
.../index/CompressedValueContainer.java | 3 +-
.../invertedindex/index/RawTableRecord.java | 4 +-
.../apache/kylin/invertedindex/index/Slice.java | 3 +-
.../kylin/invertedindex/index/SliceBuilder.java | 4 +-
.../kylin/invertedindex/index/TableRecord.java | 4 +-
.../invertedindex/index/TableRecordInfo.java | 4 +-
.../index/TableRecordInfoDigest.java | 4 +-
.../invertedindex/measure/FixedHLLCodec.java | 2 +-
.../measure/FixedLenMeasureCodec.java | 2 +-
.../measure/FixedPointLongCodec.java | 4 +-
.../invertedindex/model/IIKeyValueCodec.java | 4 +-
.../invertedindex/util/IIDictionaryBuilder.java | 2 +-
.../kylin/invertedindex/IIInstanceTest.java | 2 +-
.../invertedindex/InvertedIndexLocalTest.java | 2 +-
.../measure/FixedPointLongCodecTest.java | 2 +-
.../common/coprocessor/AggregationCache.java | 2 +-
.../common/coprocessor/FilterDecorator.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 2 +-
.../observer/AggregationScanner.java | 2 +-
.../observer/ObserverAggregationCache.java | 2 +-
.../observer/ObserverAggregators.java | 4 +-
.../v1/coprocessor/observer/ObserverTuple.java | 2 +-
.../hbase/cube/v2/CubeTupleConverter.java | 2 +-
.../endpoint/BitMapFilterEvaluator.java | 2 +-
.../endpoint/ClearTextDictionary.java | 2 +-
.../endpoint/EndpointAggregationCache.java | 2 +-
.../endpoint/EndpointAggregators.java | 6 +-
.../ii/coprocessor/endpoint/IIEndpoint.java | 4 +-
.../coprocessor/endpoint/LocalDictionary.java | 2 +-
.../storage/hbase/steps/CreateHTableJob.java | 2 +-
.../storage/hbase/steps/CubeHFileMapper.java | 2 +-
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../storage/hbase/steps/KeyValueCreator.java | 2 +-
.../storage/hbase/steps/RowValueDecoder.java | 6 +-
.../observer/AggregateRegionObserverTest.java | 2 +-
.../endpoint/BitMapFilterEvaluatorTest.java | 3 +-
.../endpoint/EndpointAggregationTest.java | 4 +-
.../hbase/steps/CubeHFileMapper2Test.java | 2 +-
.../hbase/steps/RowValueDecoderTest.java | 4 +-
189 files changed, 3343 insertions(+), 3259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 896cb63..9b2aa34 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -37,7 +37,6 @@ import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.CubeInstance;
@@ -45,6 +44,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
index 04a5b5b..4ce223e 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
@@ -5,8 +5,8 @@ import java.util.List;
import java.util.Random;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
deleted file mode 100644
index f19ce14..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/BigDecimalSerializer.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
-
- private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
-
- final DataType type;
- final int maxLength;
-
- int avoidVerbose = 0;
-
- public BigDecimalSerializer(DataType type) {
- this.type = type;
- // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
- this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
- }
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- if (value.scale() > type.getScale()) {
- if (avoidVerbose % 10000 == 0) {
- logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
- }
- value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
- }
- byte[] bytes = value.unscaledValue().toByteArray();
- if (bytes.length + 2 > maxLength) {
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- @SuppressWarnings("unused")
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
- int len = in.position() - mark + n;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
deleted file mode 100644
index 36fad00..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/DataType.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DataType implements Serializable {
-
- // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
- public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
- + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
- + "|date|time|datetime|timestamp|byte|int|short|long|string";
-
- private static final String TYPE_PATTEN_TAIL = "\\s*" //
- + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";
-
- private static final Pattern TYPE_PATTERN = Pattern.compile( //
- "(" + VALID_TYPES_STRING + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
-
- private static final Pattern CUSTOM_TYPE_PATTERN = Pattern.compile( //
- "(" + ".*?" + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
-
- public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
- public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
- public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
- public static final Set<String> STRING_FAMILY = new HashSet<String>();
- private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
- static {
- INTEGER_FAMILY.add("tinyint");
- INTEGER_FAMILY.add("smallint");
- INTEGER_FAMILY.add("integer");
- INTEGER_FAMILY.add("bigint");
-
- NUMBER_FAMILY.addAll(INTEGER_FAMILY);
- NUMBER_FAMILY.add("float");
- NUMBER_FAMILY.add("double");
- NUMBER_FAMILY.add("decimal");
- NUMBER_FAMILY.add("real");
- NUMBER_FAMILY.add("numeric");
-
- DATETIME_FAMILY.add("date");
- DATETIME_FAMILY.add("time");
- DATETIME_FAMILY.add("datetime");
- DATETIME_FAMILY.add("timestamp");
-
- STRING_FAMILY.add("varchar");
- STRING_FAMILY.add("char");
-
- LEGACY_TYPE_MAP.put("byte", "tinyint");
- LEGACY_TYPE_MAP.put("int", "integer");
- LEGACY_TYPE_MAP.put("short", "smallint");
- LEGACY_TYPE_MAP.put("long", "bigint");
- LEGACY_TYPE_MAP.put("string", "varchar");
- LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
- LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
- LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
- LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
- LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
- }
-
- private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
-
- public static final DataType ANY = DataType.getType("any");
-
- public static DataType getType(String type) {
- return getTypeInner(type, false);
- }
-
- public static DataType getCustomType(String type) {
- return getTypeInner(type, true);
- }
-
- private static DataType getTypeInner(String type, boolean isCustom) {
- if (type == null)
- return null;
-
- DataType dataType = new DataType(type, isCustom);
- DataType cached = CACHE.get(dataType);
- if (cached == null) {
- CACHE.put(dataType, dataType);
- cached = dataType;
- }
- return cached;
- }
-
- // ============================================================================
-
- private String name;
- private int precision;
- private int scale;
-
- DataType(String datatype, boolean isCustom) {
- datatype = datatype.trim().toLowerCase();
- datatype = replaceLegacy(datatype);
-
- Pattern pattern = isCustom ? CUSTOM_TYPE_PATTERN : TYPE_PATTERN;
- Matcher m = pattern.matcher(datatype);
- if (m.matches() == false)
- throw new IllegalArgumentException("bad data type -- " + datatype + ", does not match " + pattern);
-
- name = replaceLegacy(m.group(1));
- precision = -1;
- scale = -1;
-
- String leftover = m.group(2);
- if (leftover != null) {
- String[] parts = leftover.split("\\s*,\\s*");
- for (int i = 0; i < parts.length; i++) {
- int n;
- try {
- n = Integer.parseInt(parts[i]);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric");
- }
- if (i == 0)
- precision = n;
- else if (i == 1)
- scale = n;
- else
- throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts");
- }
- }
-
- // FIXME 256 for unknown string precision
- if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
- precision = 256; // to save memory at frontend, e.g. tableau will
- // allocate memory according to this
- }
-
- // FIXME (19,4) for unknown decimal precision
- if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) {
- precision = 19;
- scale = 4;
- }
- }
-
- private String replaceLegacy(String str) {
- String replace = LEGACY_TYPE_MAP.get(str);
- return replace == null ? str : replace;
- }
-
- public int getStorageBytesEstimate() {
- return DataTypeSerializer.create(this).getStorageBytesEstimate();
- }
-
- public boolean isStringFamily() {
- return STRING_FAMILY.contains(name);
- }
-
- public boolean isIntegerFamily() {
- return INTEGER_FAMILY.contains(name);
- }
-
- public boolean isNumberFamily() {
- return NUMBER_FAMILY.contains(name);
- }
-
- public boolean isDateTimeFamily() {
- return DATETIME_FAMILY.contains(name);
- }
-
- public boolean isDate() {
- return name.equals("date");
- }
-
- public boolean isTime() {
- return name.equals("time");
- }
-
- public boolean isTimestamp() {
- return name.equals("timestamp");
- }
-
- public boolean isDatetime() {
- return name.equals("datetime");
- }
-
- public boolean isTinyInt() {
- return name.equals("tinyint");
- }
-
- public boolean isSmallInt() {
- return name.equals("smallint");
- }
-
- public boolean isInt() {
- return name.equals("integer");
- }
-
- public boolean isBigInt() {
- return name.equals("bigint");
- }
-
- public boolean isFloat() {
- return name.equals("float");
- }
-
- public boolean isDouble() {
- return name.equals("double");
- }
-
- public boolean isDecimal() {
- return name.equals("decimal");
- }
-
- public boolean isHLLC() {
- return name.equals("hllc");
- }
-
- public String getName() {
- return name;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public int getScale() {
- return scale;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + precision;
- result = prime * result + scale;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- DataType other = (DataType) obj;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (precision != other.precision)
- return false;
- if (scale != other.scale)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- if (precision < 0 && scale < 0)
- return name;
- else if (scale < 0)
- return name + "(" + precision + ")";
- else
- return name + "(" + precision + "," + scale + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
deleted file mode 100644
index 99fa3fd..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/DataTypeSerializer.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.common.util.BytesSerializer;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- *
- * Note: the implementations MUST be thread-safe.
- */
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
-
- final static Map<String, Class<?>> implementations;
- static {
- HashMap<String, Class<?>> impl = Maps.newHashMap();
- impl.put("varchar", StringSerializer.class);
- impl.put("decimal", BigDecimalSerializer.class);
- impl.put("double", DoubleSerializer.class);
- impl.put("float", DoubleSerializer.class);
- impl.put("bigint", LongSerializer.class);
- impl.put("long", LongSerializer.class);
- impl.put("integer", LongSerializer.class);
- impl.put("int", LongSerializer.class);
- impl.put("smallint", LongSerializer.class);
- impl.put("date", DateTimeSerializer.class);
- impl.put("datetime", DateTimeSerializer.class);
- impl.put("timestamp", DateTimeSerializer.class);
- implementations = Collections.unmodifiableMap(impl);
- }
-
- public static boolean hasRegistered(String dataTypeName) {
- return implementations.containsKey(dataTypeName);
- }
-
- public static void register(String dataTypeName, Class<? extends DataTypeSerializer<?>> impl) {
- implementations.put(dataTypeName, impl);
- }
-
- public static DataTypeSerializer<?> create(String dataType) {
- return create(DataType.getType(dataType));
- }
-
- public static DataTypeSerializer<?> create(DataType type) {
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No DataTypeSerializer for type " + type);
-
- try {
- return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** peek into buffer and return the length of serialization */
- abstract public int peekLength(ByteBuffer in);
-
- /** return the max number of bytes to the longest serialization */
- abstract public int maxLength();
-
- /** get an estimate of size in bytes of the serialized data */
- abstract public int getStorageBytesEstimate();
-
- /** convert from String to obj (string often come as byte[] in mapred) */
- abstract public T valueOf(byte[] value);
-
- /** convert from String to obj */
- public T valueOf(String value) {
- try {
- return valueOf(value.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** convert from obj to string */
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
deleted file mode 100644
index d3ef7cd..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/DateTimeSerializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.kylin.common.datatype;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DateFormat;
-
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public DateTimeSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(DateFormat.stringToMillis(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
deleted file mode 100644
index a32fbd8..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleMutable.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
-
- private double v;
-
- public DoubleMutable() {
- this(0);
- }
-
- public DoubleMutable(double v) {
- set(v);
- }
-
- public double get() {
- return v;
- }
-
- public void set(double v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DoubleMutable)) {
- return false;
- }
- DoubleMutable other = (DoubleMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) Double.doubleToLongBits(v);
- }
-
- @Override
- public int compareTo(DoubleMutable o) {
- return (v < o.v ? -1 : (v == o.v ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Double.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
deleted file mode 100644
index f128576..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/DoubleSerializer.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-
-/**
- */
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
-
- public DoubleSerializer(DataType type) {
- }
-
- @Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
- }
-
- @Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public DoubleMutable valueOf(byte[] value) {
- DoubleMutable d = current();
- if (value == null)
- d.set(0d);
- else
- d.set(Double.parseDouble(Bytes.toString(value)));
- return d;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java b/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
deleted file mode 100644
index 72b540c..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/LongMutable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class LongMutable implements Comparable<LongMutable>, Serializable {
-
- private long v;
-
- public LongMutable() {
- this(0);
- }
-
- public LongMutable(long v) {
- set(v);
- }
-
- public long get() {
- return v;
- }
-
- public void set(long v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof LongMutable)) {
- return false;
- }
- LongMutable other = (LongMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) v;
- }
-
- @Override
- public int compareTo(LongMutable o) {
- long thisValue = this.v;
- long thatValue = o.v;
- return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Long.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
deleted file mode 100644
index 5c5e10f..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/LongSerializer.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.datatype;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- */
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public LongSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- BytesUtil.readVLong(in);
- int len = in.position() - mark;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return 9; // vlong: 1 + 8
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 5;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(Long.parseLong(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java b/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
deleted file mode 100644
index b3562eb..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/datatype/StringSerializer.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.kylin.common.datatype;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-
-public class StringSerializer extends DataTypeSerializer<String> {
-
- final DataType type;
- final int maxLength;
-
- public StringSerializer(DataType type) {
- this.type = type;
- // see serialize(): 2 byte length, rest is String.toBytes()
- this.maxLength = 2 + type.getPrecision();
- }
-
- @Override
- public void serialize(String value, ByteBuffer out) {
- int start = out.position();
-
- BytesUtil.writeUTFString(value, out);
-
- if (out.position() - start > maxLength)
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- @Override
- public String deserialize(ByteBuffer in) {
- return BytesUtil.readUTFString(in);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return BytesUtil.peekByteArrayLength(in);
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return maxLength;
- }
-
- @Override
- public String valueOf(byte[] value) {
- return Bytes.toString(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
new file mode 100644
index 0000000..6d3fa62
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.kylin.common.persistence.Writable;
+
+/**
+ * A bi-way dictionary that maps from dimension/column values to IDs and vice
+ * versa. By storing IDs instead of real values, the size of cube is
+ * significantly reduced.
+ *
+ * - IDs are smallest integers possible for the cardinality of a column, for the
+ * purpose of minimal storage space - IDs preserve ordering of values, such that
+ * range query can be applied to IDs directly
+ *
+ * A dictionary once built, is immutable. This allows optimal memory footprint
+ * by e.g. flatten the Trie structure into a byte array, replacing node pointers
+ * with array offsets.
+ *
+ * @author yangli9
+ */
+@SuppressWarnings("serial")
+abstract public class Dictionary<T> implements Writable, Serializable {
+
+ public static final byte NULL = (byte) 0xff;
+
+ // ID with all bit-1 (0xff e.g.) reserved for NULL value
+ public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
+
+ abstract public int getMinId();
+
+ abstract public int getMaxId();
+
+ public int getSize() {
+ return getMaxId() - getMinId() + 1;
+ }
+
+ /**
+ * @return the size of an ID in bytes, determined by the cardinality of column
+ */
+ abstract public int getSizeOfId();
+
+ /**
+ * @return the (maximum) size of value in bytes, determined by the longest value
+ */
+ abstract public int getSizeOfValue();
+
+ /**
+ * @return true if each entry of this dict is contained by the dict in param
+ */
+ abstract public boolean contains(Dictionary<?> another);
+
+ /**
+ * Convenient form of <code>getIdFromValue(value, 0)</code>
+ */
+ final public int getIdFromValue(T value) throws IllegalArgumentException {
+ return getIdFromValue(value, 0);
+ }
+
+ /**
+ * Returns the ID integer of given value. In case of not found
+ * <p>
+ * - if roundingFlag=0, throw IllegalArgumentException; <br>
+ * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
+ * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
+ * <p>
+ * The implementation often has cache, thus faster than the byte[] version getIdFromValueBytes()
+ *
+ * @throws IllegalArgumentException
+ * if value is not found in dictionary and rounding is off;
+ * or if rounding cannot find a smaller or bigger ID
+ */
+ final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
+ if (isNullObjectForm(value))
+ return nullId();
+ else
+ return getIdFromValueImpl(value, roundingFlag);
+ }
+
+ final public boolean containsValue(T value) throws IllegalArgumentException {
+ if (isNullObjectForm(value)) {
+ return true;
+ } else {
+ try {
+ //if no key found, it will throw exception
+ getIdFromValueImpl(value, 0);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ return true;
+ }
+ }
+
+ protected boolean isNullObjectForm(T value) {
+ return value == null;
+ }
+
+ abstract protected int getIdFromValueImpl(T value, int roundingFlag);
+
+ /**
+ * @return the value corresponds to the given ID
+ * @throws IllegalArgumentException
+ * if ID is not found in dictionary
+ */
+ final public T getValueFromId(int id) throws IllegalArgumentException {
+ if (isNullId(id))
+ return null;
+ else
+ return getValueFromIdImpl(id);
+ }
+
+ abstract protected T getValueFromIdImpl(int id);
+
+ /**
+ * Convenient form of
+ * <code>getIdFromValueBytes(value, offset, len, 0)</code>
+ */
+ final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException {
+ return getIdFromValueBytes(value, offset, len, 0);
+ }
+
+ /**
+ * A lower level API, return ID integer from raw value bytes. In case of not found
+ * <p>
+ * - if roundingFlag=0, throw IllegalArgumentException; <br>
+ * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
+ * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
+ * <p>
+ * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
+ *
+ * @throws IllegalArgumentException
+ * if value is not found in dictionary and rounding is off;
+ * or if rounding cannot find a smaller or bigger ID
+ */
+ final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
+ if (isNullByteForm(value, offset, len))
+ return nullId();
+ else {
+ int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
+ if (id < 0)
+ throw new IllegalArgumentException("Value not exists!");
+ return id;
+ }
+ }
+
+ protected boolean isNullByteForm(byte[] value, int offset, int len) {
+ return value == null;
+ }
+
+ abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
+
+ final public byte[] getValueBytesFromId(int id) {
+ if (isNullId(id))
+ return BytesUtil.EMPTY_BYTE_ARRAY;
+ else
+ return getValueBytesFromIdImpl(id);
+ }
+
+ abstract protected byte[] getValueBytesFromIdImpl(int id);
+
+ /**
+ * A lower level API, get byte values from ID, return the number of bytes
+ * written. Bypassing the cache layer, this could be significantly slower
+ * than getIdFromValue(T value).
+ *
+ * @return size of value bytes, 0 if empty string, -1 if null
+ *
+ * @throws IllegalArgumentException
+ * if ID is not found in dictionary
+ */
+ final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
+ if (isNullId(id))
+ return -1;
+ else
+ return getValueBytesFromIdImpl(id, returnValue, offset);
+ }
+
+ abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
+
+ abstract public void dump(PrintStream out);
+
+ public int nullId() {
+ return NULL_ID[getSizeOfId()];
+ }
+
+ public boolean isNullId(int id) {
+ int nullId = NULL_ID[getSizeOfId()];
+ return (nullId & id) == nullId;
+ }
+
+ /** utility that converts a dictionary ID to string, preserving order */
+ public static String dictIdToString(byte[] idBytes, int offset, int length) {
+ try {
+ return new String(idBytes, offset, length, "ISO-8859-1");
+ } catch (UnsupportedEncodingException e) {
+ // never happen
+ return null;
+ }
+ }
+
+ /** the reverse of dictIdToString(), returns integer ID */
+ public static int stringToDictId(String str) {
+ try {
+ byte[] bytes = str.getBytes("ISO-8859-1");
+ return BytesUtil.readUnsigned(bytes, 0, bytes.length);
+ } catch (UnsupportedEncodingException e) {
+ // never happen
+ return 0;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java b/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
deleted file mode 100644
index cba6795..0000000
--- a/core-common/src/test/java/org/apache/kylin/common/datatype/BigDecimalSerializerTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.kylin.common.datatype;
-
-import static org.junit.Assert.*;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- */
-public class BigDecimalSerializerTest {
-
- private static BigDecimalSerializer bigDecimalSerializer;
-
- @BeforeClass
- public static void beforeClass() {
- bigDecimalSerializer = new BigDecimalSerializer(DataType.getType("decimal"));
- }
-
- @Test
- public void testNormal() {
- BigDecimal input = new BigDecimal("1234.1234");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input, output);
- }
-
- @Test
- public void testScaleOutOfRange() {
- BigDecimal input = new BigDecimal("1234.1234567890");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testOutOfPrecision() {
- BigDecimal input = new BigDecimal("66855344214907231736.4924");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- bigDecimalSerializer.serialize(input, buffer);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java b/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
deleted file mode 100644
index 81345b7..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/AggregationType.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.aggregation.basic.BasicAggregationFactory;
-import org.apache.kylin.aggregation.hllc.HLLCAggregationFactory;
-import org.apache.kylin.aggregation.topn.TopNAggregationFactory;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Maps;
-
-abstract public class AggregationType {
-
- private static final Map<String, IAggregationFactory> factoryRegistry = Maps.newConcurrentMap();
- private static final IAggregationFactory defaultFactory = new BasicAggregationFactory();
-
- static {
- factoryRegistry.put(FunctionDesc.FUNC_COUNT_DISTINCT, new HLLCAggregationFactory());
- factoryRegistry.put(FunctionDesc.FUNC_TOP_N, new TopNAggregationFactory());
- }
-
- public static AggregationType create(String funcName, String dataType) {
- funcName = funcName.toUpperCase();
- dataType = dataType.toLowerCase();
-
- IAggregationFactory factory = factoryRegistry.get(funcName);
- if (factory == null)
- factory = defaultFactory;
-
- AggregationType result = factory.createAggregationType(funcName, dataType);
-
- // register serializer for aggr data type
- DataType aggregationDataType = result.getAggregationDataType();
- if (DataTypeSerializer.hasRegistered(aggregationDataType.getName()) == false) {
- DataTypeSerializer.register(aggregationDataType.getName(), result.getAggregationDataSeralizer());
- }
-
- return result;
- }
-
- /* ============================================================================
- * Define
- * ---------------------------------------------------------------------------- */
-
- abstract public DataType getAggregationDataType();
-
- abstract public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer();
-
- abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
-
- /* ============================================================================
- * Build
- * ---------------------------------------------------------------------------- */
-
- abstract public MeasureAggregator<?> newAggregator();
-
- abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
-
- abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
-
- /* ============================================================================
- * Cube Selection
- * ---------------------------------------------------------------------------- */
-
- /* ============================================================================
- * Query
- * ---------------------------------------------------------------------------- */
-
- /* ============================================================================
- * Storage
- * ---------------------------------------------------------------------------- */
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
deleted file mode 100644
index cbdd9da..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/IAggregationFactory.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.apache.kylin.aggregation;
-
-public interface IAggregationFactory {
-
- public AggregationType createAggregationType(String funcName, String dataType);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
deleted file mode 100644
index 6968a9d..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregator.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation;
-
-import java.io.Serializable;
-
-/**
- */
-@SuppressWarnings("serial")
-abstract public class MeasureAggregator<V> implements Serializable {
-
- public static MeasureAggregator<?> create(String funcName, String dataType) {
- return AggregationType.create(funcName, dataType).newAggregator();
- }
-
- public static int guessBigDecimalMemBytes() {
- // 116 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 8 // ref to BigDecimal
- + 8 // BigDecimal obj shell
- + 100; // guess of BigDecimal internal
- }
-
- public static int guessDoubleMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to DoubleWritable
- + 8 // DoubleWritable obj shell
- + 8; // size of double
- */
- }
-
- public static int guessLongMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to LongWritable
- + 8 // LongWritable obj shell
- + 8; // size of long
- */
- }
-
- // ============================================================================
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- }
-
- abstract public void reset();
-
- abstract public void aggregate(V value);
-
- abstract public V getState();
-
- // get an estimate of memory consumption UPPER BOUND
- abstract public int getMemBytesEstimate();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
deleted file mode 100644
index 3aa575b..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureAggregators.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- */
-@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
-public class MeasureAggregators implements Serializable {
-
- private final MeasureAggregator[] aggs;
- private final int descLength;
-
- public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureAggregators(MeasureDesc... measureDescs) {
- descLength = measureDescs.length;
- aggs = new MeasureAggregator[descLength];
-
- Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
- for (int i = 0; i < descLength; i++) {
- FunctionDesc func = measureDescs[i].getFunction();
- aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
- measureIndexMap.put(measureDescs[i].getName(), i);
- }
- // fill back dependent aggregator
- for (int i = 0; i < descLength; i++) {
- String depMsrRef = measureDescs[i].getDependentMeasureRef();
- if (depMsrRef != null) {
- int index = measureIndexMap.get(depMsrRef);
- aggs[i].setDependentAggregator(aggs[index]);
- }
- }
- }
-
- public void reset() {
- for (int i = 0; i < aggs.length; i++) {
- aggs[i].reset();
- }
- }
-
- public void aggregate(Object[] values) {
- assert values.length == descLength;
-
- for (int i = 0; i < descLength; i++) {
- aggs[i].aggregate(values[i]);
- }
- }
-
- public void collectStates(Object[] states) {
- for (int i = 0; i < descLength; i++) {
- states[i] = aggs[i].getState();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java b/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
deleted file mode 100644
index 8721954..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/MeasureCodec.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MeasureCodec {
-
- int nMeasures;
- DataTypeSerializer[] serializers;
-
- public MeasureCodec(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureCodec(MeasureDesc... measureDescs) {
- String[] dataTypes = new String[measureDescs.length];
- for (int i = 0; i < dataTypes.length; i++) {
- dataTypes[i] = measureDescs[i].getFunction().getReturnType();
- }
- init(dataTypes);
- }
-
- public MeasureCodec(String... dataTypes) {
- init(dataTypes);
- }
-
- private void init(String[] dataTypes) {
- nMeasures = dataTypes.length;
- serializers = new DataTypeSerializer[nMeasures];
-
- for (int i = 0; i < nMeasures; i++) {
- serializers[i] = DataTypeSerializer.create(dataTypes[i]);
- }
- }
-
- public DataTypeSerializer getSerializer(int idx) {
- return serializers[idx];
- }
-
- public void decode(ByteBuffer buf, Object[] result) {
- assert result.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- result[i] = serializers[i].deserialize(buf);
- }
- }
-
- public void encode(Object[] values, ByteBuffer out) {
- assert values.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- serializers[i].serialize(values[i], out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
deleted file mode 100644
index 7ea88bd..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregation.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.util.List;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.BigDecimalSerializer;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.common.datatype.DateTimeSerializer;
-import org.apache.kylin.common.datatype.DoubleSerializer;
-import org.apache.kylin.common.datatype.LongSerializer;
-import org.apache.kylin.common.datatype.StringSerializer;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public class BasicAggregation extends AggregationType {
-
- private final String funcName;
- private final DataType dataType;
-
- public BasicAggregation(String funcName, String dataType) {
- this.funcName = funcName;
- this.dataType = DataType.getType(dataType);
- }
-
- @Override
- public DataType getAggregationDataType() {
- return dataType;
- }
-
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- if (dataType.isStringFamily())
- return StringSerializer.class;
- else if (dataType.isIntegerFamily())
- return LongSerializer.class;
- else if (dataType.isDecimal())
- return BigDecimalSerializer.class;
- else if (dataType.isNumberFamily())
- return DoubleSerializer.class;
- else if (dataType.isDateTimeFamily())
- return DateTimeSerializer.class;
- else
- throw new IllegalArgumentException("No default serializer for type " + dataType);
- }
-
- @Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public MeasureAggregator<?> newAggregator() {
- if (isSum() || isCount()) {
- if (dataType.isDecimal())
- return new BigDecimalSumAggregator();
- else if (dataType.isIntegerFamily())
- return new LongSumAggregator();
- else if (dataType.isNumberFamily())
- return new DoubleSumAggregator();
- } else if (isMax()) {
- if (dataType.isDecimal())
- return new BigDecimalMaxAggregator();
- else if (dataType.isIntegerFamily())
- return new LongMaxAggregator();
- else if (dataType.isNumberFamily())
- return new DoubleMaxAggregator();
- } else if (isMin()) {
- if (dataType.isDecimal())
- return new BigDecimalMinAggregator();
- else if (dataType.isIntegerFamily())
- return new LongMinAggregator();
- else if (dataType.isNumberFamily())
- return new DoubleMinAggregator();
- }
- throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + dataType + "'");
- }
-
- private boolean isSum() {
- return FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName);
- }
-
- private boolean isCount() {
- return FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName);
- }
-
- private boolean isMax() {
- return FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName);
- }
-
- private boolean isMin() {
- return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
- }
-
- @Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
deleted file mode 100644
index 547d45f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BasicAggregationFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.IAggregationFactory;
-
-public class BasicAggregationFactory implements IAggregationFactory {
-
- @Override
- public AggregationType createAggregationType(String funcName, String dataType) {
- return new BasicAggregation(funcName, dataType);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
deleted file mode 100644
index ca044d0..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMaxAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.math.BigDecimal;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) < 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
deleted file mode 100644
index 3c3c85e..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalMinAggregator.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.math.BigDecimal;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
deleted file mode 100644
index 19aef3c..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/BigDecimalSumAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.math.BigDecimal;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal sum = new BigDecimal(0);
-
- @Override
- public void reset() {
- sum = new BigDecimal(0);
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- sum = sum.add(value);
- }
-
- @Override
- public BigDecimal getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
deleted file mode 100644
index 99896a6..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMaxAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DoubleMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
[06/21] kylin git commit: KYLIN-976 AggregationType interface,
serializer and aggregator
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
deleted file mode 100644
index 202596d..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSerializer.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public LongSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- BytesUtil.readVLong(in);
- int len = in.position() - mark;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return 9; // vlong: 1 + 8
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 5;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(Long.parseLong(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
index c85c83c..38d728a 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.basic;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.common.datatype.LongMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
deleted file mode 100644
index e84278d..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/StringSerializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.apache.kylin.aggregation.basic;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-
-public class StringSerializer extends DataTypeSerializer<String> {
-
- final DataType type;
- final int maxLength;
-
- public StringSerializer(DataType type) {
- this.type = type;
- // see serialize(): 2 byte length, rest is String.toBytes()
- this.maxLength = 2 + type.getPrecision();
- }
-
- @Override
- public void serialize(String value, ByteBuffer out) {
- int start = out.position();
-
- BytesUtil.writeUTFString(value, out);
-
- if (out.position() - start > maxLength)
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- @Override
- public String deserialize(ByteBuffer in) {
- return BytesUtil.readUTFString(in);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return BytesUtil.peekByteArrayLength(in);
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return maxLength;
- }
-
- @Override
- public String valueOf(byte[] value) {
- return Bytes.toString(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
new file mode 100644
index 0000000..d5ceba5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.hllc;
+
+import java.util.List;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class HLLCAggregation extends AggregationType {
+
+ private final DataType dataType;
+
+ public HLLCAggregation(String dataType) {
+ this.dataType = DataType.getType(dataType);
+
+ if (this.dataType.getPrecision() < 10 || this.dataType.getPrecision() > 16)
+ throw new IllegalArgumentException("HLLC precision must be between 10 and 16");
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ return HLLCSerializer.class;
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ if (dataType.isHLLC())
+ return new HLLCAggregator(dataType.getPrecision());
+ else
+ return new LDCAggregator();
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
new file mode 100644
index 0000000..18c021d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.hllc;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.IAggregationFactory;
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+public class HLLCAggregationFactory implements IAggregationFactory {
+
+ @Override
+ public AggregationType createAggregationType(String funcName, String dataType) {
+ if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName) == false)
+ throw new IllegalArgumentException();
+
+ return new HLLCAggregation(dataType);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
index f7804f4..5612892 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
@@ -21,9 +21,9 @@ package org.apache.kylin.aggregation.hllc;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
index 643bcae..151c1ee 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
@@ -19,7 +19,7 @@
package org.apache.kylin.aggregation.hllc;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.common.datatype.LongMutable;
/**
* Long Distinct Count
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
new file mode 100644
index 0000000..251abd9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.topn;
+
+import java.util.List;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.hllc.HLLCSerializer;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class TopNAggregation extends AggregationType {
+
+ private final DataType dataType;
+
+ public TopNAggregation(String dataType) {
+ this.dataType = DataType.getType(dataType);
+
+ if (this.dataType.getPrecision() < 1 || this.dataType.getPrecision() > 1000)
+ throw new IllegalArgumentException("TopN precision must be between 1 and 1000");
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ return HLLCSerializer.class;
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ return new TopNAggregator();
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
new file mode 100644
index 0000000..1ea22c8
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.aggregation.topn;
+
+import org.apache.kylin.aggregation.AggregationType;
+import org.apache.kylin.aggregation.IAggregationFactory;
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+public class TopNAggregationFactory implements IAggregationFactory {
+
+ @Override
+ public AggregationType createAggregationType(String funcName, String dataType) {
+ if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName) == false)
+ throw new IllegalArgumentException();
+
+ return new TopNAggregation(dataType);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
index 8c44f8f..8088842 100644
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
+++ b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
@@ -18,17 +18,17 @@
package org.apache.kylin.aggregation.topn;
-import org.apache.kylin.aggregation.DataTypeSerializer;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.DoubleDeltaSerializer;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
/**
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index b88f9df..3619d69 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -6,9 +6,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
-import org.apache.kylin.aggregation.DataTypeSerializer;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.aggregation.basic.StringSerializer;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
+import org.apache.kylin.common.datatype.StringSerializer;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 33db11b..aa0a530 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -11,7 +11,6 @@ import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 428137a..8d132e2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -6,11 +6,11 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index 26f1636..d30186e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -23,9 +23,8 @@ package org.apache.kylin.cube.gridtable;
import java.nio.ByteBuffer;
import java.util.Map;
-import org.apache.kylin.aggregation.DataTypeSerializer;
import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.aggregation.basic.StringSerializer;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 8c6146b..a393179 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -28,9 +28,9 @@ import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.kylin.common.datatype.DoubleMutable;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.DoubleMutable;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index 951c054..bf4278a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -20,8 +20,8 @@ package org.apache.kylin.cube.inmemcubing;
import com.google.common.base.Preconditions;
import org.apache.kylin.aggregation.MeasureCodec;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
index d7a48d9..a21fe9f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
@@ -21,7 +21,7 @@ package org.apache.kylin.cube.kv;
import java.util.Collection;
import java.util.Comparator;
-import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.common.datatype.DataType;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 5f400d7..e4307e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -26,13 +26,13 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index e3d3640..229c679 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -5,10 +5,10 @@ import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedList;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.cube.util.KryoUtils;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
public class GTInfo {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index be92f73..2783f55 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -2,8 +2,8 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
-import org.apache.kylin.aggregation.DataTypeSerializer;
import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.util.ImmutableBitSet;
@SuppressWarnings({ "rawtypes", "unchecked" })
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index 9852acc..f8d7f30 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -22,11 +22,11 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.metadata.model.DataType;
public class UnitTestSupport {
@@ -48,11 +48,11 @@ public class UnitTestSupport {
Builder builder = GTInfo.builder();
builder.setCodeSystem(new GTSampleCodeSystem());
builder.setColumns( //
- DataType.getInstance("varchar(10)"), //
- DataType.getInstance("varchar(10)"), //
- DataType.getInstance("varchar(10)"), //
- DataType.getInstance("bigint"), //
- DataType.getInstance("decimal") //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("bigint"), //
+ DataType.getType("decimal") //
);
builder.setPrimaryKey(setOf(0));
builder.setColumnPreferIndex(setOf(0));
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
deleted file mode 100644
index 27aa07c..0000000
--- a/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.apache.kylin.aggregation.basic;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.aggregation.basic.BigDecimalSerializer;
-import org.apache.kylin.metadata.model.DataType;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- */
-public class BigDecimalSerializerTest {
-
- private static BigDecimalSerializer bigDecimalSerializer;
-
- @BeforeClass
- public static void beforeClass() {
- bigDecimalSerializer = new BigDecimalSerializer(DataType.getInstance("decimal"));
- }
-
- @Test
- public void testNormal() {
- BigDecimal input = new BigDecimal("1234.1234");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input, output);
- }
-
- @Test
- public void testScaleOutOfRange() {
- BigDecimal input = new BigDecimal("1234.1234567890");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testOutOfPrecision() {
- BigDecimal input = new BigDecimal("66855344214907231736.4924");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- bigDecimalSerializer.serialize(input, buffer);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
index 8ee7b8d..8ae44b6 100644
--- a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
@@ -1,22 +1,21 @@
package org.apache.kylin.aggregation.topn;
-import org.apache.kylin.aggregation.topn.TopNCounterSerializer;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
import org.junit.Assert;
import org.junit.Test;
-import java.nio.ByteBuffer;
-
/**
*
*/
public class TopNCounterSerializerTest {
- private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getInstance("topn(10)"));
+ private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getType("topn(10)"));
@Test
public void testSerialization() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index f407977..93f0419 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -28,10 +28,10 @@ import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
import org.apache.kylin.aggregation.basic.LongSumAggregator;
import org.apache.kylin.aggregation.hllc.HLLCAggregator;
+import org.apache.kylin.common.datatype.DoubleMutable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DoubleMutable;
-import org.apache.kylin.common.util.LongMutable;
import org.junit.Test;
public class AggregationCacheMemSizeTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index f5247e2..b3981e8 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -17,7 +17,7 @@
package org.apache.kylin.gridtable;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.math.BigDecimal;
@@ -27,9 +27,10 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.Dictionary;
@@ -46,7 +47,6 @@ import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.Before;
@@ -408,11 +408,11 @@ public class DictGridTableTest {
Builder builder = GTInfo.builder();
builder.setCodeSystem(newDictCodeSystem());
builder.setColumns( //
- DataType.getInstance("timestamp"), //
- DataType.getInstance("integer"), //
- DataType.getInstance("varchar(10)"), //
- DataType.getInstance("bigint"), //
- DataType.getInstance("decimal") //
+ DataType.getType("timestamp"), //
+ DataType.getType("integer"), //
+ DataType.getType("varchar(10)"), //
+ DataType.getType("bigint"), //
+ DataType.getType("decimal") //
);
builder.setPrimaryKey(setOf(0, 1));
builder.setColumnPreferIndex(setOf(0));
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 361617a..4313f4b 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -26,8 +26,8 @@ import java.math.BigDecimal;
import java.util.BitSet;
import java.util.List;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index a964e67..7f4da61 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -17,29 +17,28 @@
package org.apache.kylin.gridtable;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
+import it.uniroma3.mat.extendedset.intset.ConciseSet;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import org.apache.kylin.aggregation.basic.StringSerializer;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
+import org.apache.kylin.common.datatype.StringSerializer;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.Test;
import com.google.common.collect.Lists;
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
public class SimpleInvertedIndexTest {
GTInfo info;
@@ -167,7 +166,7 @@ public class SimpleInvertedIndexTest {
public static ConstantTupleFilter constFilter(int id) {
byte[] space = new byte[10];
ByteBuffer buf = ByteBuffer.wrap(space);
- StringSerializer stringSerializer = new StringSerializer(DataType.getInstance("string"));
+ StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
stringSerializer.serialize("" + id, buf);
ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
return new ConstantTupleFilter(data);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 0e13d7e..4da5471 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -24,9 +24,9 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.kylin.aggregation.MeasureCodec;
+import org.apache.kylin.common.datatype.DoubleMutable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.util.DoubleMutable;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 8027f41..300c240 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -21,20 +21,19 @@ package org.apache.kylin.dict;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.source.ReadableTable;
-import org.apache.kylin.source.ReadableTable.TableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
/**
* @author yangli9
@@ -105,7 +104,7 @@ public class DictionaryGenerator {
logger.debug("Building dictionary object " + JsonUtil.writeValueAsString(info));
columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), info.getSourceColumnIndex());
- return buildDictionaryFromValueEnumerator(DataType.getInstance(info.getDataType()), columnValueEnumerator);
+ return buildDictionaryFromValueEnumerator(DataType.getType(info.getDataType()), columnValueEnumerator);
} finally {
if (columnValueEnumerator != null)
columnValueEnumerator.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 2a1304a..f3cab64 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -30,10 +30,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
@@ -218,7 +218,7 @@ public class DictionaryManager {
logger.info("Use one of the merging dictionaries directly");
return dicts.get(0);
} else {
- Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(DataType.getInstance(newDictInfo.getDataType()), dicts);
+ Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts);
return trySaveNewDict(newDict, newDictInfo);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index e51153c..8c8dcfc 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -18,10 +18,9 @@
package org.apache.kylin.dict;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
-import java.io.IOException;
+import java.io.IOException;
import java.math.BigDecimal;
import java.util.Collection;
import java.util.Collections;
@@ -29,8 +28,8 @@ import java.util.List;
import java.util.Random;
import java.util.Set;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -54,7 +53,7 @@ public class NumberDictionaryTest {
}
// check "" is treated as NULL, not a code of dictionary
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getInstance("integer"), new IterableDictionaryValueEnumerator(intBytes));
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getType("integer"), new IterableDictionaryValueEnumerator(intBytes));
assertEquals(4, dict.getSize());
final int id = ((NumberDictionary<String>) dict).getIdFromValue("");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
index e4bdf3d..0e687f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TimeConditionLiteralsReplacer.java
@@ -5,8 +5,8 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 6162477..f03e736 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -18,14 +18,15 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.io.Serializable;
-
/**
* Column Metadata from Source. All name should be uppercase.
* <p/>
@@ -72,7 +73,7 @@ public class ColumnDesc implements Serializable {
public void setDatatype(String datatype) {
//logger.info("setting datatype to " + datatype);
this.datatype = datatype;
- type = DataType.getInstance(datatype);
+ type = DataType.getType(datatype);
}
public String getId() {
@@ -132,7 +133,7 @@ public class ColumnDesc implements Serializable {
if (id != null)
zeroBasedIndex = Integer.parseInt(id) - 1;
- DataType normalized = DataType.getInstance(datatype);
+ DataType normalized = DataType.getType(datatype);
if (normalized == null) {
this.setDatatype(null);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
deleted file mode 100644
index 1333426..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.model;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.aggregation.DataTypeSerializer;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DataType implements Serializable {
-
- public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
- + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
- + "|date|time|datetime|timestamp|byte|int|short|long|string|hllc|topn" //
- + "|" + TblColRef.InnerDataTypeEnum.LITERAL.getDataType() //
- + "|" + TblColRef.InnerDataTypeEnum.DERIVED.getDataType();
-
- private static final Pattern TYPE_PATTERN = Pattern.compile(
- // standard sql types, ref:
- // http://www.w3schools.com/sql/sql_datatypes_general.asp
- "(" + VALID_TYPES_STRING + ")" + "\\s*" //
- + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?", Pattern.CASE_INSENSITIVE);
-
- public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
- public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
- public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
- public static final Set<String> STRING_FAMILY = new HashSet<String>();
- private static final Set<Integer> HLLC_PRECISIONS = new HashSet<Integer>();
- private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
- static {
- INTEGER_FAMILY.add("tinyint");
- INTEGER_FAMILY.add("smallint");
- INTEGER_FAMILY.add("integer");
- INTEGER_FAMILY.add("bigint");
-
- NUMBER_FAMILY.addAll(INTEGER_FAMILY);
- NUMBER_FAMILY.add("float");
- NUMBER_FAMILY.add("double");
- NUMBER_FAMILY.add("decimal");
- NUMBER_FAMILY.add("real");
- NUMBER_FAMILY.add("numeric");
-
- DATETIME_FAMILY.add("date");
- DATETIME_FAMILY.add("time");
- DATETIME_FAMILY.add("datetime");
- DATETIME_FAMILY.add("timestamp");
-
- STRING_FAMILY.add("varchar");
- STRING_FAMILY.add("char");
-
- LEGACY_TYPE_MAP.put("byte", "tinyint");
- LEGACY_TYPE_MAP.put("int", "integer");
- LEGACY_TYPE_MAP.put("short", "smallint");
- LEGACY_TYPE_MAP.put("long", "bigint");
- LEGACY_TYPE_MAP.put("string", "varchar");
- LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
- LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
- LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
- LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
- LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
-
- for (int i = 10; i <= 16; i++)
- HLLC_PRECISIONS.add(i);
- }
-
- private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
-
- public static final DataType ANY = DataType.getInstance("any");
-
- public static DataType getInstance(String type) {
- if (type == null)
- return null;
-
- DataType dataType = new DataType(type);
- DataType cached = CACHE.get(dataType);
- if (cached == null) {
- CACHE.put(dataType, dataType);
- cached = dataType;
- }
- return cached;
- }
-
- // ============================================================================
-
- private String name;
- private int precision;
- private int scale;
-
- DataType(String datatype) {
- parseDataType(datatype);
- }
-
- private void parseDataType(String datatype) {
- datatype = datatype.trim().toLowerCase();
- datatype = replaceLegacy(datatype);
-
- Matcher m = TYPE_PATTERN.matcher(datatype);
- if (m.matches() == false)
- throw new IllegalArgumentException("bad data type -- " + datatype + ", does not match " + TYPE_PATTERN);
-
- name = replaceLegacy(m.group(1));
- precision = -1;
- scale = -1;
-
- String leftover = m.group(2);
- if (leftover != null) {
- String[] parts = leftover.split("\\s*,\\s*");
- for (int i = 0; i < parts.length; i++) {
- int n;
- try {
- n = Integer.parseInt(parts[i]);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric");
- }
- if (i == 0)
- precision = n;
- else if (i == 1)
- scale = n;
- else
- throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts");
- }
- }
-
- // FIXME 256 for unknown string precision
- if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
- precision = 256; // to save memory at frontend, e.g. tableau will
- // allocate memory according to this
- }
-
- // FIXME (19,4) for unknown decimal precision
- if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) {
- precision = 19;
- scale = 4;
- }
-
- if (isHLLC() && HLLC_PRECISIONS.contains(precision) == false)
- throw new IllegalArgumentException("HLLC precision must be one of " + HLLC_PRECISIONS);
- }
-
- private String replaceLegacy(String str) {
- String replace = LEGACY_TYPE_MAP.get(str);
- return replace == null ? str : replace;
- }
-
- public int getStorageBytesEstimate() {
- return DataTypeSerializer.create(this).getStorageBytesEstimate();
- }
-
- public boolean isStringFamily() {
- return STRING_FAMILY.contains(name);
- }
-
- public boolean isIntegerFamily() {
- return INTEGER_FAMILY.contains(name);
- }
-
- public boolean isNumberFamily() {
- return NUMBER_FAMILY.contains(name);
- }
-
- public boolean isDateTimeFamily() {
- return DATETIME_FAMILY.contains(name);
- }
-
- public boolean isDate() {
- return name.equals("date");
- }
-
- public boolean isTime() {
- return name.equals("time");
- }
-
- public boolean isTimestamp() {
- return name.equals("timestamp");
- }
-
- public boolean isDatetime() {
- return name.equals("datetime");
- }
-
- public boolean isTinyInt() {
- return name.equals("tinyint");
- }
-
- public boolean isSmallInt() {
- return name.equals("smallint");
- }
-
- public boolean isInt() {
- return name.equals("integer");
- }
-
- public boolean isBigInt() {
- return name.equals("bigint");
- }
-
- public boolean isFloat() {
- return name.equals("float");
- }
-
- public boolean isDouble() {
- return name.equals("double");
- }
-
- public boolean isDecimal() {
- return name.equals("decimal");
- }
-
- public boolean isHLLC() {
- return name.equals("hllc");
- }
-
- public boolean isTopN() {
- return name.equals("topn");
- }
-
- public String getName() {
- return name;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public int getScale() {
- return scale;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + precision;
- result = prime * result + scale;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- DataType other = (DataType) obj;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (precision != other.precision)
- return false;
- if (scale != other.scale)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- if (precision < 0 && scale < 0)
- return name;
- else if (scale < 0)
- return name + "(" + precision + ")";
- else
- return name + "(" + precision + "," + scale + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index b8cefa2..d22d0a3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -21,6 +21,8 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
import java.util.Collection;
+import org.apache.kylin.common.datatype.DataType;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -53,7 +55,7 @@ public class FunctionDesc {
public void init(TableDesc factTable) {
expression = expression.toUpperCase();
- returnDataType = DataType.getInstance(returnType);
+ returnDataType = DataType.getType(returnType);
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
p.setValue(p.getValue().toUpperCase());
@@ -69,6 +71,11 @@ public class FunctionDesc {
}
parameter.setColRefs(colRefs);
+
+ // make sure sum/max/min returns the exact type as its input
+ if (isSum() || isMax() || isMin() && (colRefs.size() > 0)) {
+ setReturnType(colRefs.get(0).getDatatype());
+ }
}
public String getRewriteFieldName() {
@@ -183,7 +190,7 @@ public class FunctionDesc {
public void setReturnType(String returnType) {
this.returnType = returnType;
- this.returnDataType = DataType.getInstance(returnType);
+ this.returnDataType = DataType.getType(returnType);
}
public TblColRef selectTblColRef(Collection<TblColRef> metricColumns, String factTableName) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index ddb1e1a..de27145 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -18,10 +18,11 @@
package org.apache.kylin.metadata.model;
-import org.apache.commons.lang.StringUtils;
-
import java.io.Serializable;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.datatype.DataType;
+
/**
*/
@SuppressWarnings("serial")
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
index a4cdcbe..4a8c5d1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigestUtil.java
@@ -1,12 +1,12 @@
package org.apache.kylin.metadata.realization;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Function;
@@ -52,9 +52,9 @@ public class SQLDigestUtil {
//ts column type differentiate
private static String formatTimeStr(DataType type, long ts) {
String ret;
- if (type == DataType.getInstance("date")) {
+ if (type == DataType.getType("date")) {
ret = DateFormat.formatToDateStr(ts);
- } else if (type == DataType.getInstance("long")) {
+ } else if (type == DataType.getType("long")) {
ret = String.valueOf(ts);
} else {
throw new IllegalArgumentException("Illegal type for partition column " + type);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index da009df..4f011cf 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -24,11 +24,11 @@ import java.util.List;
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
+import org.apache.kylin.common.datatype.DoubleMutable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.DoubleMutable;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 5b2d20e..55235be 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -32,8 +32,8 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.apache.kylin.aggregation.MeasureCodec;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 2acc751..5145efc 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.engine.mr.steps;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
@@ -29,14 +29,19 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.dict.*;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.source.ReadableTable.TableSignature;
@@ -71,7 +76,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
List<byte[]> values = new ArrayList<byte[]>();
values.add(new byte[] { 101, 101, 101 });
values.add(new byte[] { 102, 102, 102 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getInstance(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
dictionaryManager.trySaveNewDict(dict, newDictInfo);
((TrieDictionary) dict).dump(System.out);
@@ -123,7 +128,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
values.add(new byte[] { 99, 99, 99 });
else
values.add(new byte[] { 98, 98, 98 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getInstance(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
+ Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
dictionaryManager.trySaveNewDict(dict, newDictInfo);
((TrieDictionary) dict).dump(System.out);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index 11b1897..aad1fa2 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -21,9 +21,9 @@ package org.apache.kylin.invertedindex.index;
import java.util.Arrays;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 71d7bae..12fcf29 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -22,8 +22,8 @@ import java.util.Arrays;
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.dict.Dictionary;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 27519bc..d9e5b26 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -20,12 +20,12 @@ package org.apache.kylin.invertedindex.index;
import java.util.List;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -71,10 +71,10 @@ public class TableRecordInfo {
isMetric[i] = desc.isMetricsCol(i);
dataTypes[i] = tblColRef.getDatatype();
if (isMetric[i]) {
- lengths[i] = FixedLenMeasureCodec.get(DataType.getInstance(tblColRef.getColumnDesc().getDatatype())).getLength();
+ lengths[i] = FixedLenMeasureCodec.get(DataType.getType(tblColRef.getColumnDesc().getDatatype())).getLength();
} else {
if (Array.isEmpty(dictionaryMap)) {
- final DataType dataType = DataType.getInstance(tblColRef.getColumnDesc().getDatatype());
+ final DataType dataType = DataType.getType(tblColRef.getColumnDesc().getDatatype());
if (dataType.isNumberFamily()) {
lengths[i] = 16;
} else if (dataType.isStringFamily()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index bd27e38..afb3633 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -21,11 +21,11 @@ package org.apache.kylin.invertedindex.index;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.DataType;
import com.google.common.base.Objects;
@@ -55,7 +55,7 @@ public class TableRecordInfoDigest {
this.measureCodecs = new FixedLenMeasureCodec[nColumns];
for (int i = 0; i < isMetric.length; i++) {
if (isMetric[i]) {
- measureCodecs[i] = FixedLenMeasureCodec.get(DataType.getInstance(metricDataTypes[i]));
+ measureCodecs[i] = FixedLenMeasureCodec.get(DataType.getType(metricDataTypes[i]));
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
index de35f91..42e5185 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
@@ -19,8 +19,8 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
index 35872be..4fa8714 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
@@ -18,10 +18,10 @@
package org.apache.kylin.invertedindex.measure;
-import org.apache.kylin.metadata.model.DataType;
-
import java.nio.ByteBuffer;
+import org.apache.kylin.common.datatype.DataType;
+
abstract public class FixedLenMeasureCodec<T> {
public static FixedLenMeasureCodec<?> get(DataType type) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
index a5d74f7..2b65192 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
@@ -20,9 +20,9 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 4dd1723..0f29d95 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Iterator;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
@@ -33,7 +34,6 @@ import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.DataType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -111,11 +111,11 @@ public class IIKeyValueCodec implements KeyValueCodec {
final boolean emptyDictionary = Array.isEmpty(dictionaries);
for (int i = 0; i < nColumns; ++i) {
if (isMetric[i]) {
- final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
+ final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getType(dataTypes[i]));
lengths[i] = fixedLenMeasureCodec.getLength();
} else {
if (emptyDictionary) {
- final DataType dataType = DataType.getInstance(dataTypes[i]);
+ final DataType dataType = DataType.getType(dataTypes[i]);
if (dataType.isNumberFamily()) {
lengths[i] = 16;
} else if (dataType.isStringFamily()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
index 27a4f71..600d034 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
@@ -1,7 +1,6 @@
package org.apache.kylin.invertedindex.measure;
-import org.apache.kylin.invertedindex.measure.FixedPointLongCodec;
-import org.apache.kylin.metadata.model.DataType;
+import org.apache.kylin.common.datatype.DataType;
import org.junit.Test;
/**
@@ -10,35 +9,35 @@ public class FixedPointLongCodecTest {
@Test
public void testEncode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
long x = codec.getValueIgnoringDecimalPoint("12.12345");
org.junit.Assert.assertEquals(1212345, x);
}
@Test
public void testEncode2() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
long x = codec.getValueIgnoringDecimalPoint("12.1234");
org.junit.Assert.assertEquals(1212340, x);
}
@Test
public void testEncode3() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
long x = codec.getValueIgnoringDecimalPoint("12.123456");
org.junit.Assert.assertEquals(1212345, x);
}
@Test
public void testEncode4() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
long x = codec.getValueIgnoringDecimalPoint("12");
org.junit.Assert.assertEquals(1200000, x);
}
@Test
public void testDecode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)"));
String x = codec.restoreDecimalPoint(1212345);
org.junit.Assert.assertEquals("12.12345", x);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 8f0b17b..910ed31 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -23,15 +23,15 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.invertedindex.index.RawTableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
@@ -128,7 +128,7 @@ public class EndpointAggregators {
this.metricValues = new Object[funcNames.length];
this.measureSerializers = new FixedLenMeasureCodec[funcNames.length];
for (int i = 0; i < this.measureSerializers.length; ++i) {
- this.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(dataTypes[i]));
+ this.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getType(dataTypes[i]));
}
}
[02/21] kylin git commit: KYLIN-976 very initial
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/StringSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/StringSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/StringSerializer.java
deleted file mode 100644
index db27ca0..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/StringSerializer.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-
-public class StringSerializer extends DataTypeSerializer<String> {
-
- final DataType type;
- final int maxLength;
-
- public StringSerializer(DataType type) {
- this.type = type;
- // see serialize(): 2 byte length, rest is String.toBytes()
- this.maxLength = 2 + type.getPrecision();
- }
-
- @Override
- public void serialize(String value, ByteBuffer out) {
- int start = out.position();
-
- BytesUtil.writeUTFString(value, out);
-
- if (out.position() - start > maxLength)
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- @Override
- public String deserialize(ByteBuffer in) {
- return BytesUtil.readUTFString(in);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return BytesUtil.peekByteArrayLength(in);
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return maxLength;
- }
-
- @Override
- public String valueOf(byte[] value) {
- return Bytes.toString(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
deleted file mode 100644
index 468d077..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializer.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.DoubleDeltaSerializer;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- *
- */
-public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteArray>> {
-
- private DoubleDeltaSerializer dds = new DoubleDeltaSerializer(3);
-
- private int precision;
-
- public TopNCounterSerializer(DataType dataType) {
- this.precision = dataType.getPrecision();
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- @SuppressWarnings("unused")
- int capacity = in.getInt();
- int size = in.getInt();
- int keyLength = in.getInt();
- dds.deserialize(in);
- int len = in.position() - mark + keyLength * size;
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8);
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return precision * TopNCounter.EXTRA_SPACE_RATE * 8;
- }
-
- @Override
- public TopNCounter<ByteArray> valueOf(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- int sizeOfId = buffer.getInt();
- int keyEncodedValue = buffer.getInt();
- double counter = buffer.getDouble();
-
- ByteArray key = new ByteArray(sizeOfId);
- BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, sizeOfId);
-
- TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(precision * TopNCounter.EXTRA_SPACE_RATE);
- topNCounter.offer(key, counter);
- return topNCounter;
- }
-
- @Override
- public void serialize(TopNCounter<ByteArray> value, ByteBuffer out) {
- double[] counters = value.getCounters();
- List<ByteArray> peek = value.peek(1);
- int keyLength = peek.size() > 0 ? peek.get(0).length() : 0;
- out.putInt(value.getCapacity());
- out.putInt(value.size());
- out.putInt(keyLength);
- dds.serialize(counters, out);
- Iterator<Counter<ByteArray>> iterator = value.iterator();
- while (iterator.hasNext()) {
- out.put(iterator.next().getItem().array());
- }
- }
-
- @Override
- public TopNCounter<ByteArray> deserialize(ByteBuffer in) {
- int capacity = in.getInt();
- int size = in.getInt();
- int keyLength = in.getInt();
- double[] counters = dds.deserialize(in);
-
- TopNCounter<ByteArray> counter = new TopNCounter<ByteArray>(capacity);
- ByteArray byteArray;
- for (int i = 0; i < size; i++) {
- byteArray = new ByteArray(keyLength);
- in.get(byteArray.array());
- counter.offerToHead(byteArray, counters[i]);
- }
-
- return counter;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index 590ef3d..1333426 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
+import org.apache.kylin.aggregation.DataTypeSerializer;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
deleted file mode 100644
index 6ba7115..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodecTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import org.apache.kylin.metadata.model.DataType;
-import org.junit.Test;
-
-/**
- */
-public class FixedPointLongCodecTest {
-
- @Test
- public void testEncode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.12345");
- org.junit.Assert.assertEquals(1212345, x);
- }
-
- @Test
- public void testEncode2() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.1234");
- org.junit.Assert.assertEquals(1212340, x);
- }
-
- @Test
- public void testEncode3() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12.123456");
- org.junit.Assert.assertEquals(1212345, x);
- }
-
- @Test
- public void testEncode4() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
- long x = codec.getValueIgnoringDecimalPoint("12");
- org.junit.Assert.assertEquals(1200000, x);
- }
-
- @Test
- public void testDecode1() {
- FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
- String x = codec.restoreDecimalPoint(1212345);
- org.junit.Assert.assertEquals("12.12345", x);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializerTest.java
deleted file mode 100644
index 682bc24..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializerTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.apache.kylin.metadata.measure.serializer;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.metadata.model.DataType;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- */
-public class BigDecimalSerializerTest {
-
- private static BigDecimalSerializer bigDecimalSerializer;
-
- @BeforeClass
- public static void beforeClass() {
- bigDecimalSerializer = new BigDecimalSerializer(DataType.getInstance("decimal"));
- }
-
- @Test
- public void testNormal() {
- BigDecimal input = new BigDecimal("1234.1234");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input, output);
- }
-
- @Test
- public void testScaleOutOfRange() {
- BigDecimal input = new BigDecimal("1234.1234567890");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- buffer.mark();
- bigDecimalSerializer.serialize(input, buffer);
- buffer.reset();
- BigDecimal output = bigDecimalSerializer.deserialize(buffer);
- assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testOutOfPrecision() {
- BigDecimal input = new BigDecimal("66855344214907231736.4924");
- ByteBuffer buffer = ByteBuffer.allocate(256);
- bigDecimalSerializer.serialize(input, buffer);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializerTest.java
deleted file mode 100644
index 3c88446..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/measure/serializer/TopNCounterSerializerTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package org.apache.kylin.metadata.measure.serializer;
-
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-
-/**
- *
- */
-public class TopNCounterSerializerTest {
-
- private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getInstance("topn(10)"));
-
- @SuppressWarnings("unchecked")
- @Test
- public void testSerialization() {
- TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
- Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
- for (Integer i : stream) {
- vs.offer(new ByteArray(Bytes.toBytes(i)));
- }
-
- ByteBuffer out = ByteBuffer.allocate(1024);
- serializer.serialize(vs, out);
-
- byte[] copyBytes = new byte[out.position()];
- System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
-
- ByteBuffer in = ByteBuffer.wrap(copyBytes);
- TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
-
- Assert.assertEquals(vs.toString(), vsNew.toString());
-
- }
-
- @Test
- public void testValueOf() {
-
- TopNCounter<ByteArray> origin = new TopNCounter<ByteArray>(10);
- ByteArray key = new ByteArray(1);
- ByteBuffer byteBuffer = key.asBuffer();
- BytesUtil.writeVLong(20l, byteBuffer);
- origin.offer(key, 1.0);
-
- byteBuffer = ByteBuffer.allocate(1024);
- byteBuffer.putInt(1);
- byteBuffer.putInt(20);
- byteBuffer.putDouble(1.0);
- TopNCounter<ByteArray> counter = serializer.valueOf(byteBuffer.array());
-
-
- Assert.assertEquals(origin.toString(), counter.toString());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 11b03bd..da009df 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -27,8 +27,8 @@ import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.measure.LongMutable;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 557e325..3fa7d5c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -7,7 +7,9 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Maps;
+
import org.apache.hadoop.io.Text;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSplitter;
@@ -24,7 +26,6 @@ import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index f7b9e9a..0f94dca 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -23,6 +23,8 @@ import java.nio.ByteBuffer;
import java.util.List;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.aggregation.MeasureAggregators;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
@@ -30,8 +32,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureAggregators;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index db254f6..ab87b21 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -3,6 +3,8 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.List;
+import org.apache.kylin.aggregation.MeasureAggregators;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -14,8 +16,6 @@ import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureAggregators;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 22324ba..72ce84c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
@@ -47,7 +48,6 @@ import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index b5a63cb..e49e1a6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
@@ -47,7 +48,6 @@ import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index fac3ff6..5b2d20e 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -31,13 +31,13 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 32d4d2a..4159718 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.aggregation.MeasureAggregators;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
@@ -56,15 +58,11 @@ import org.apache.kylin.cube.model.*;
import org.apache.kylin.cube.util.CubingUtils;
import org.apache.kylin.dict.*;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.engine.spark.cube.BufferedCuboidWriter;
import org.apache.kylin.engine.spark.cube.DefaultTupleConverter;
import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.metadata.measure.MeasureAggregators;
-import org.apache.kylin.metadata.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -84,9 +82,11 @@ import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.hive.HiveContext;
import org.reflections.Reflections;
+
import scala.Tuple2;
import javax.annotation.Nullable;
+
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index c126fdd..11b1897 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -23,9 +23,9 @@ import java.util.Arrays;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 2521fbf..71d7bae 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -23,9 +23,9 @@ import java.util.Arrays;
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.measure.LongMutable;
/**
* TableRecord extends RawTableRecord by decorating it with a TableRecordInfo.
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index 9627187..27519bc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index 2b2c514..bd27e38 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -23,8 +23,8 @@ import java.util.Arrays;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import com.google.common.base.Objects;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
new file mode 100644
index 0000000..de35f91
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.invertedindex.measure;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.model.DataType;
+
+/**
+ */
+public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
+
+ private DataType type;
+ private int presision;
+ private HyperLogLogPlusCounter current;
+
+ public FixedHLLCodec(DataType type) {
+ this.type = type;
+ this.presision = type.getPrecision();
+ this.current = new HyperLogLogPlusCounter(this.presision);
+ }
+
+ @Override
+ public int getLength() {
+ return 1 << presision;
+ }
+
+ @Override
+ public DataType getDataType() {
+ return type;
+ }
+
+ @Override
+ public HyperLogLogPlusCounter valueOf(String value) {
+ current.clear();
+ if (value == null)
+ current.add("__nUlL__");
+ else
+ current.add(value.getBytes());
+ return current;
+ }
+
+ @Override
+ public Object getValue() {
+ return current;
+ }
+
+ @Override
+ public HyperLogLogPlusCounter read(byte[] buf, int offset) {
+ current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ return current;
+ }
+
+ @Override
+ public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
+ v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ }
+
+ @Override
+ public HyperLogLogPlusCounter read(ByteBuffer buffer) {
+ current.readRegistersArray(buffer);
+ return current;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
new file mode 100644
index 0000000..35872be
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex.measure;
+
+import org.apache.kylin.metadata.model.DataType;
+
+import java.nio.ByteBuffer;
+
+abstract public class FixedLenMeasureCodec<T> {
+
+ public static FixedLenMeasureCodec<?> get(DataType type) {
+ if (type.isHLLC()) {
+ return new FixedHLLCodec(type);
+ } else {
+ return new FixedPointLongCodec(type);
+ }
+ }
+
+ abstract public int getLength();
+
+ abstract public DataType getDataType();
+
+ abstract public T valueOf(String value);
+
+ abstract public Object getValue();
+
+ abstract public T read(byte[] buf, int offset);
+
+ abstract public void write(T v, byte[] buf, int offset);
+
+ abstract public T read(ByteBuffer buffer);
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
new file mode 100644
index 0000000..a5d74f7
--- /dev/null
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex.measure;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.LongMutable;
+import org.apache.kylin.metadata.model.DataType;
+
+public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
+
+ private static final int SIZE = 8;
+ // number of digits after decimal point
+ int scale;
+ DataType type;
+ // avoid massive object creation
+ LongMutable current = new LongMutable();
+
+ public FixedPointLongCodec(DataType type) {
+ this.type = type;
+ this.scale = Math.max(0, type.getScale());
+ }
+
+ @Override
+ public int getLength() {
+ return SIZE;
+ }
+
+ @Override
+ public DataType getDataType() {
+ return type;
+ }
+
+ long getValueIgnoringDecimalPoint(String value) {
+ int index = value.indexOf('.');
+
+ if (index == 0 || index == value.length() - 1) {
+ throw new RuntimeException("Bad decimal format: " + value);
+ } else if (index < 0) {
+ return Long.valueOf(value) * (int) Math.pow(10, scale);
+ } else {
+ StringBuilder sb = new StringBuilder();
+ sb.append(value.substring(0, index));
+
+ //if there are more than scale digits after the decimal point, the tail will be discarded
+ int end = Math.min(value.length(), index + scale + 1);
+ sb.append(value.substring(index + 1, end));
+ int diff = index + scale + 1 - value.length();
+ //if there are less than scale digits after the decimal point, the tail will be compensated
+ for (int i = 0; i < diff; i++) {
+ sb.append('0');
+ }
+ return Long.valueOf(sb.toString());
+ }
+ }
+
+ String restoreDecimalPoint(long value) {
+ if (scale < 0) {
+ throw new RuntimeException("Bad scale: " + scale + " with value: " + value);
+ } else if (scale == 0) {
+ return Long.toString(value);
+ } else {
+ return String.format("%." + scale + "f", value / (Math.pow(10, scale)));
+ }
+ }
+
+ @Override
+ public LongMutable valueOf(String value) {
+ if (value == null)
+ current.set(0L);
+ else
+ current.set(getValueIgnoringDecimalPoint(value));
+ return current;
+ }
+
+ @Override
+ public String getValue() {
+ if (scale == 0)
+ return current.toString();
+ else
+ return restoreDecimalPoint(current.get());
+ }
+
+ @Override
+ public LongMutable read(byte[] buf, int offset) {
+ current.set(BytesUtil.readLong(buf, offset, SIZE));
+ return current;
+ }
+
+ @Override
+ public void write(LongMutable v, byte[] buf, int offset) {
+ BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE);
+ }
+
+ @Override
+ public LongMutable read(ByteBuffer buffer) {
+ current.set(BytesUtil.readLong(buffer, SIZE));
+ return current;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index e17133f..4dd1723 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -32,7 +32,7 @@ import org.apache.kylin.invertedindex.index.ColumnValueContainer;
import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
+import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
new file mode 100644
index 0000000..8ae3dfd
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.invertedindex.IIDescManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class IIDescManagerTest extends LocalFileMetadataTestCase {
+
+ public static final String TEST_II_DESC_NAME = "test_kylin_ii_left_join_desc";
+
+ @Before
+ public void setup() {
+ createTestMetadata();
+ }
+
+ @After
+ public void clean() {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCRUD() throws IOException {
+ IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
+
+ String newDescName = "Copy of " + TEST_II_DESC_NAME;
+
+ try {
+ IIDesc testRecord = mgr.getIIDesc(newDescName);
+ if (testRecord != null)
+ mgr.removeIIDesc(testRecord);
+ } catch (IOException e) {
+ // just ensure the old one is removed
+ }
+
+ Assert.assertNull(mgr.getIIDesc(newDescName));
+ IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
+
+ desc.setName(newDescName);
+ desc.setLastModified(0);
+
+ mgr.createIIDesc(desc);
+
+ desc = mgr.getIIDesc(newDescName);
+
+ Assert.assertNotNull(desc);
+
+ mgr.updateIIDesc(desc); // this will trigger cache wipe; please ignore the HTTP error in logs.
+
+ mgr.removeIIDesc(desc);
+
+ Assert.assertNull(mgr.getIIDesc(newDescName));
+
+ }
+
+ @Test
+ public void testReload() throws IOException {
+ IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
+
+ IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
+
+ // do some modification
+ desc.setUuid(UUID.randomUUID().toString());
+
+ IIDesc newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
+
+ Assert.assertEquals(desc, newDesc);
+
+ // reload the cache
+ mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
+
+ newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
+
+ Assert.assertNotEquals(desc, newDesc);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
new file mode 100644
index 0000000..aac915d
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.invertedindex.IIDescManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class IIDescTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setup() {
+ this.createTestMetadata();
+
+ }
+
+ @After
+ public void clear() {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGetIIDesc() {
+
+ IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
+ DataModelDesc model = iiDesc.getModel();
+ Assert.assertNotNull(iiDesc);
+ Assert.assertNotNull(model);
+
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+ IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
+ String str = JsonUtil.writeValueAsIndentString(iiDesc);
+ System.out.println(str);
+ IIDesc desc2 = JsonUtil.readValue(str, IIDesc.class);
+
+ Assert.assertEquals(iiDesc, desc2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
new file mode 100644
index 0000000..f30a876
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.invertedindex.IIDescManager;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class IIInstanceTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setup() {
+ createTestMetadata();
+ }
+
+ @After
+ public void clean() {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGetIIsByDesc() throws IOException {
+ IIManager mgr = IIManager.getInstance(getTestConfig());
+
+ List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_left_join_desc");
+
+ Assert.assertTrue(iiInstances.size() > 0);
+
+ }
+
+ @Test
+ public void testCreateIIInstance() throws IOException {
+
+ IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
+
+ IIInstance ii = IIInstance.create("new ii", "default", iiDesc);
+
+ IIManager iiMgr = IIManager.getInstance(getTestConfig());
+
+ List<IIInstance> allIIList = iiMgr.listAllIIs();
+
+ iiMgr.createII(ii);
+
+ Assert.assertNotNull(iiMgr.getII("new ii"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
new file mode 100644
index 0000000..d72899b
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.ShardingSliceBuilder;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecord;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.invertedindex.model.IIRow;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.Sets;
+
+public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
+
+ IIInstance ii;
+ TableRecordInfo info;
+ List<String> lines;
+ private Dictionary<?>[] dictionaryMap;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+
+ File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
+ FileInputStream in = new FileInputStream(file);
+ this.lines = IOUtils.readLines(in, "UTF-8");
+ in.close();
+
+ dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() {
+ @Nullable
+ @Override
+ public List<String> apply(@Nullable String input) {
+ return Lists.newArrayList(input.split(","));
+ }
+ }), ii.getDescriptor());
+ this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap);
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCompressedValueContainer() {
+ // create container
+ CompressedValueContainer container = new CompressedValueContainer(info.getDigest(), 0, 500);
+ Dictionary<String> dict = info.dict(0);
+
+ byte[] buf = new byte[dict.getSizeOfId()];
+ ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
+
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
+ container.append(bytes);
+ }
+ BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
+ container.append(bytes);
+ container.closeForChange();
+
+ // copy by serialization
+ ImmutableBytesWritable copy = container.toBytes();
+ CompressedValueContainer container2 = new CompressedValueContainer(info.getDigest(), 0, 500);
+ container2.fromBytes(copy);
+
+ // check the copy
+ int i = 0;
+ for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
+ container2.getValueAt(i++, bytes);
+ int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
+ assertEquals(v, value);
+ }
+ container2.getValueAt(i++, bytes);
+ int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
+ assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
+ assertEquals(container, container2);
+ }
+
+ @Test
+ public void testCodec() throws IOException {
+ List<TableRecord> records = loadRecordsSorted();
+ System.out.println(records.size() + " records");
+ List<Slice> slices = buildTimeSlices(records);
+ System.out.println(slices.size() + " slices");
+
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+ List<IIRow> kvs = encodeKVs(codec, slices);
+ System.out.println(kvs.size() + " KV pairs");
+
+ List<Slice> slicesCopy = decodeKVs(codec, kvs);
+ assertEquals(slices.size(), slicesCopy.size());
+ for (int i = 0; i < slices.size(); i++) {
+ assertEquals(slices.get(i), slicesCopy.get(i));
+ }
+
+ List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
+ assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
+ dump(recordsCopy);
+ }
+
+ private Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException{
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+ Set<TblColRef> dimensionColumns = Sets.newHashSet();
+ for (int i = 0; i < desc.listAllColumns().size(); i++) {
+ if (!desc.isMetricsCol(i)) {
+ dimensionColumns.add(desc.listAllColumns().get(i));
+ }
+ }
+ for (List<String> row : table) {
+ for (int i = 0; i < row.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(desc.listAllColumns().get(i), cell);
+ }
+ }
+ Dictionary<?>[] result = new Dictionary<?>[desc.listAllColumns().size()];
+ for (TblColRef tblColRef : valueMap.keys()) {
+ result[desc.findColumn(tblColRef)] = DictionaryGenerator.buildDictionaryFromValueEnumerator(tblColRef.getType(),
+ new IterableDictionaryValueEnumerator(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ return input.getBytes();
+ }
+ })));
+ }
+ return result;
+ }
+
+ private List<TableRecord> loadRecordsSorted() throws IOException {
+ List<TableRecord> records = Lists.newArrayList();
+ for (String line : lines) {
+ String[] fields = line.split(",");
+ TableRecord rec = info.createTableRecord();
+ for (int col = 0; col < fields.length; col++) {
+ rec.setValueString(col, fields[col]);
+ }
+ records.add(rec);
+ }
+
+ Collections.sort(records, new Comparator<TableRecord>() {
+ @Override
+ public int compare(TableRecord a, TableRecord b) {
+ long x = a.getTimestamp() - b.getTimestamp();
+ if (x > 0)
+ return 1;
+ else if (x == 0)
+ return 0;
+ else
+ return -1;
+ }
+ });
+
+ return records;
+ }
+
+ private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
+ ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
+ List<Slice> slices = Lists.newArrayList();
+ for (TableRecord rec : records) {
+ //here assume there less records than slice size for each shard
+ Slice slice = builder.append(rec);
+ if (slice != null) {
+ slice.setLocalDictionaries(dictionaryMap);
+ slices.add(slice);
+ }
+ }
+ List<Slice> finals = builder.close();
+ for (Slice slice : finals) {
+ slice.setLocalDictionaries(dictionaryMap);
+ }
+ slices.addAll(finals);
+
+ Collections.sort(slices);
+ return slices;
+ }
+
+ private List<IIRow> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
+
+ List<IIRow> kvs = Lists.newArrayList();
+ for (Slice slice : slices) {
+ kvs.addAll(codec.encodeKeyValue(slice));
+ }
+ return kvs;
+ }
+
+ private List<Slice> decodeKVs(IIKeyValueCodec codec, List<IIRow> kvs) {
+ List<Slice> slices = Lists.newArrayList();
+ for (Slice slice : codec.decodeKeyValue(kvs)) {
+ slices.add(slice);
+ }
+ return slices;
+ }
+
+ private List<TableRecord> iterateRecords(List<Slice> slices) {
+ List<TableRecord> records = Lists.newArrayList();
+ for (Slice slice : slices) {
+ for (RawTableRecord rec : slice) {
+ records.add(new TableRecord((RawTableRecord) rec.clone(), info));
+ }
+ }
+ return records;
+ }
+
+ private void dump(Iterable<TableRecord> records) {
+ for (TableRecord rec : records) {
+ System.out.println(rec.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
new file mode 100644
index 0000000..943e76c
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.invertedindex;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.junit.Test;
+
+import com.ning.compress.lzf.LZFDecoder;
+import com.ning.compress.lzf.LZFEncoder;
+
+/**
+ */
+public class LZFTest {
+ @Test
+ public void test() throws IOException {
+
+ byte[] raw = new byte[] { 1, 2, 3, 3, 2, 23 };
+ byte[] data = LZFEncoder.encode(raw);
+
+ byte[] data2 = new byte[data.length * 2];
+ java.lang.System.arraycopy(data, 0, data2, 0, data.length);
+ ImmutableBytesWritable bytes = new ImmutableBytesWritable();
+ bytes.set(data2, 0, data.length);
+
+ try {
+ byte[] uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
+ } catch (IOException e) {
+ throw new RuntimeException("LZF decode failure", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
deleted file mode 100644
index e715b4f..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIDescManagerTest extends LocalFileMetadataTestCase {
-
- public static final String TEST_II_DESC_NAME = "test_kylin_ii_left_join_desc";
-
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCRUD() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- String newDescName = "Copy of " + TEST_II_DESC_NAME;
-
- try {
- IIDesc testRecord = mgr.getIIDesc(newDescName);
- if (testRecord != null)
- mgr.removeIIDesc(testRecord);
- } catch (IOException e) {
- // just ensure the old one is removed
- }
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- desc.setName(newDescName);
- desc.setLastModified(0);
-
- mgr.createIIDesc(desc);
-
- desc = mgr.getIIDesc(newDescName);
-
- Assert.assertNotNull(desc);
-
- mgr.updateIIDesc(desc); // this will trigger cache wipe; please ignore the HTTP error in logs.
-
- mgr.removeIIDesc(desc);
-
- Assert.assertNull(mgr.getIIDesc(newDescName));
-
- }
-
- @Test
- public void testReload() throws IOException {
- IIDescManager mgr = IIDescManager.getInstance(getTestConfig());
-
- IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- // do some modification
- desc.setUuid(UUID.randomUUID().toString());
-
- IIDesc newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertEquals(desc, newDesc);
-
- // reload the cache
- mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
-
- newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
-
- Assert.assertNotEquals(desc, newDesc);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
deleted file mode 100644
index 007d36f..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIDescTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() {
- this.createTestMetadata();
-
- }
-
- @After
- public void clear() {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIDesc() {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
- DataModelDesc model = iiDesc.getModel();
- Assert.assertNotNull(iiDesc);
- Assert.assertNotNull(model);
-
- }
-
- @Test
- public void testSerialization() throws IOException {
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
- String str = JsonUtil.writeValueAsIndentString(iiDesc);
- System.out.println(str);
- IIDesc desc2 = JsonUtil.readValue(str, IIDesc.class);
-
- Assert.assertEquals(iiDesc, desc2);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
deleted file mode 100644
index 8a0c2ba..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIInstanceTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.IIDescManager;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class IIInstanceTest extends LocalFileMetadataTestCase {
- @Before
- public void setup() {
- createTestMetadata();
- }
-
- @After
- public void clean() {
- cleanupTestMetadata();
- }
-
- @Test
- public void testGetIIsByDesc() throws IOException {
- IIManager mgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_left_join_desc");
-
- Assert.assertTrue(iiInstances.size() > 0);
-
- }
-
- @Test
- public void testCreateIIInstance() throws IOException {
-
- IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc");
-
- IIInstance ii = IIInstance.create("new ii", "default", iiDesc);
-
- IIManager iiMgr = IIManager.getInstance(getTestConfig());
-
- List<IIInstance> allIIList = iiMgr.listAllIIs();
-
- iiMgr.createII(ii);
-
- Assert.assertNotNull(iiMgr.getII("new ii"));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
deleted file mode 100644
index 6e05759..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/InvertedIndexLocalTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.index.CompressedValueContainer;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.ShardingSliceBuilder;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.apache.kylin.invertedindex.model.IIRow;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.SetMultimap;
-import com.google.common.collect.Sets;
-
-public class InvertedIndexLocalTest extends LocalFileMetadataTestCase {
-
- IIInstance ii;
- TableRecordInfo info;
- List<String> lines;
- private Dictionary<?>[] dictionaryMap;
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
-
- File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
- FileInputStream in = new FileInputStream(file);
- this.lines = IOUtils.readLines(in, "UTF-8");
- in.close();
-
- dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() {
- @Nullable
- @Override
- public List<String> apply(@Nullable String input) {
- return Lists.newArrayList(input.split(","));
- }
- }), ii.getDescriptor());
- this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCompressedValueContainer() {
- // create container
- CompressedValueContainer container = new CompressedValueContainer(info.getDigest(), 0, 500);
- Dictionary<String> dict = info.dict(0);
-
- byte[] buf = new byte[dict.getSizeOfId()];
- ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf);
-
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId());
- container.append(bytes);
- }
- BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId());
- container.append(bytes);
- container.closeForChange();
-
- // copy by serialization
- ImmutableBytesWritable copy = container.toBytes();
- CompressedValueContainer container2 = new CompressedValueContainer(info.getDigest(), 0, 500);
- container2.fromBytes(copy);
-
- // check the copy
- int i = 0;
- for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) {
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(v, value);
- }
- container2.getValueAt(i++, bytes);
- int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength());
- assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value);
- assertEquals(container, container2);
- }
-
- @Test
- public void testCodec() throws IOException {
- List<TableRecord> records = loadRecordsSorted();
- System.out.println(records.size() + " records");
- List<Slice> slices = buildTimeSlices(records);
- System.out.println(slices.size() + " slices");
-
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
- List<IIRow> kvs = encodeKVs(codec, slices);
- System.out.println(kvs.size() + " KV pairs");
-
- List<Slice> slicesCopy = decodeKVs(codec, kvs);
- assertEquals(slices.size(), slicesCopy.size());
- for (int i = 0; i < slices.size(); i++) {
- assertEquals(slices.get(i), slicesCopy.get(i));
- }
-
- List<TableRecord> recordsCopy = iterateRecords(slicesCopy);
- assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy));
- dump(recordsCopy);
- }
-
- private Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException{
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
- Set<TblColRef> dimensionColumns = Sets.newHashSet();
- for (int i = 0; i < desc.listAllColumns().size(); i++) {
- if (!desc.isMetricsCol(i)) {
- dimensionColumns.add(desc.listAllColumns().get(i));
- }
- }
- for (List<String> row : table) {
- for (int i = 0; i < row.size(); i++) {
- String cell = row.get(i);
- valueMap.put(desc.listAllColumns().get(i), cell);
- }
- }
- Dictionary<?>[] result = new Dictionary<?>[desc.listAllColumns().size()];
- for (TblColRef tblColRef : valueMap.keys()) {
- result[desc.findColumn(tblColRef)] = DictionaryGenerator.buildDictionaryFromValueEnumerator(tblColRef.getType(),
- new IterableDictionaryValueEnumerator(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- return input.getBytes();
- }
- })));
- }
- return result;
- }
-
- private List<TableRecord> loadRecordsSorted() throws IOException {
- List<TableRecord> records = Lists.newArrayList();
- for (String line : lines) {
- String[] fields = line.split(",");
- TableRecord rec = info.createTableRecord();
- for (int col = 0; col < fields.length; col++) {
- rec.setValueString(col, fields[col]);
- }
- records.add(rec);
- }
-
- Collections.sort(records, new Comparator<TableRecord>() {
- @Override
- public int compare(TableRecord a, TableRecord b) {
- long x = a.getTimestamp() - b.getTimestamp();
- if (x > 0)
- return 1;
- else if (x == 0)
- return 0;
- else
- return -1;
- }
- });
-
- return records;
- }
-
- private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException {
- ShardingSliceBuilder builder = new ShardingSliceBuilder(info);
- List<Slice> slices = Lists.newArrayList();
- for (TableRecord rec : records) {
- //here assume there less records than slice size for each shard
- Slice slice = builder.append(rec);
- if (slice != null) {
- slice.setLocalDictionaries(dictionaryMap);
- slices.add(slice);
- }
- }
- List<Slice> finals = builder.close();
- for (Slice slice : finals) {
- slice.setLocalDictionaries(dictionaryMap);
- }
- slices.addAll(finals);
-
- Collections.sort(slices);
- return slices;
- }
-
- private List<IIRow> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) {
-
- List<IIRow> kvs = Lists.newArrayList();
- for (Slice slice : slices) {
- kvs.addAll(codec.encodeKeyValue(slice));
- }
- return kvs;
- }
-
- private List<Slice> decodeKVs(IIKeyValueCodec codec, List<IIRow> kvs) {
- List<Slice> slices = Lists.newArrayList();
- for (Slice slice : codec.decodeKeyValue(kvs)) {
- slices.add(slice);
- }
- return slices;
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- System.out.println(rec.toString());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
deleted file mode 100644
index d39c421..0000000
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/LZFTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.invertedindex.invertedindex;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.junit.Test;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-
-/**
- */
-public class LZFTest {
- @Test
- public void test() throws IOException {
-
- byte[] raw = new byte[] { 1, 2, 3, 3, 2, 23 };
- byte[] data = LZFEncoder.encode(raw);
-
- byte[] data2 = new byte[data.length * 2];
- java.lang.System.arraycopy(data, 0, data2, 0, data.length);
- ImmutableBytesWritable bytes = new ImmutableBytesWritable();
- bytes.set(data2, 0, data.length);
-
- try {
- byte[] uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
new file mode 100644
index 0000000..27a4f71
--- /dev/null
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.invertedindex.measure;
+
+import org.apache.kylin.invertedindex.measure.FixedPointLongCodec;
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.Test;
+
+/**
+ */
+public class FixedPointLongCodecTest {
+
+ @Test
+ public void testEncode1() {
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ long x = codec.getValueIgnoringDecimalPoint("12.12345");
+ org.junit.Assert.assertEquals(1212345, x);
+ }
+
+ @Test
+ public void testEncode2() {
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ long x = codec.getValueIgnoringDecimalPoint("12.1234");
+ org.junit.Assert.assertEquals(1212340, x);
+ }
+
+ @Test
+ public void testEncode3() {
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ long x = codec.getValueIgnoringDecimalPoint("12.123456");
+ org.junit.Assert.assertEquals(1212345, x);
+ }
+
+ @Test
+ public void testEncode4() {
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ long x = codec.getValueIgnoringDecimalPoint("12");
+ org.junit.Assert.assertEquals(1200000, x);
+ }
+
+ @Test
+ public void testDecode1() {
+ FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getInstance("decimal(18,5)"));
+ String x = codec.restoreDecimalPoint(1212345);
+ org.junit.Assert.assertEquals("12.12345", x);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/836f2336/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
index 440dcc4..1a6d944 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
@@ -20,8 +20,8 @@ package org.apache.kylin.storage.hbase.common.coprocessor;
import java.util.Map;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
import com.google.common.collect.Maps;
[19/21] kylin git commit: KYLIN-976 Clean up,
remove all explict check on TopN & HLLC
Posted by li...@apache.org.
KYLIN-976 Clean up, remove all explict check on TopN & HLLC
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7f50c17a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7f50c17a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7f50c17a
Branch: refs/heads/2.x-staging
Commit: 7f50c17a5202be9508c21d9cc301d231c8b5843e
Parents: 636aeec
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Dec 4 16:38:04 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:55:39 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 16 ++-
.../apache/kylin/common/topn/TopNCounter.java | 2 +-
.../kylin/common/util/CliCommandExecutor.java | 4 +-
.../java/org/apache/kylin/common/util/Pair.java | 3 +-
.../kylin/common/topn/TopNCounterTest.java | 4 +-
.../kylin/cube/CubeCapabilityChecker.java | 5 -
.../org/apache/kylin/cube/CubeDescManager.java | 15 ++-
.../org/apache/kylin/cube/CubeInstance.java | 3 -
.../java/org/apache/kylin/cube/CubeManager.java | 4 +-
.../kylin/cube/gridtable/CubeCodeSystem.java | 2 +-
.../gridtable/CuboidToGridTableMapping.java | 30 +----
.../cube/gridtable/TrimmedCubeCodeSystem.java | 2 +-
.../InMemCubeBuilderInputConverter.java | 3 +-
.../cube/inmemcubing/InMemCubeBuilderUtils.java | 30 +----
.../org/apache/kylin/cube/model/CubeDesc.java | 11 +-
.../model/validation/rule/FunctionRule.java | 4 -
.../kylin/gridtable/GTSampleCodeSystem.java | 2 +-
.../cube/inmemcubing/InMemCubeBuilderTest.java | 25 ++--
.../apache/kylin/dict/lookup/LookupTable.java | 2 +-
.../apache/kylin/measure/IMeasureFactory.java | 24 ----
.../apache/kylin/measure/MeasureAggregator.java | 6 +-
.../kylin/measure/MeasureAggregators.java | 2 +-
.../org/apache/kylin/measure/MeasureType.java | 50 ++------
.../kylin/measure/MeasureTypeFactory.java | 89 ++++++++++++++
.../measure/basic/BasicMeasureFactory.java | 32 -----
.../kylin/measure/basic/BasicMeasureType.java | 74 +++++++-----
.../measure/hllc/HLLCAggregationFactory.java | 36 ------
.../kylin/measure/hllc/HLLCMeasureType.java | 76 ++++++++----
.../kylin/measure/hllc/HLLCSerializer.java | 3 +-
.../kylin/measure/hllc/LDCAggregator.java | 63 ----------
.../kylin/measure/topn/TopNMeasureFactory.java | 36 ------
.../kylin/measure/topn/TopNMeasureType.java | 116 +++++++++++++------
.../kylin/metadata/datatype/DataType.java | 33 ++++--
.../metadata/datatype/DataTypeSerializer.java | 38 +++---
.../kylin/metadata/model/FunctionDesc.java | 65 ++---------
.../kylin/metadata/tuple/TeeTupleIterator.java | 5 -
.../org/apache/kylin/metadata/tuple/Tuple.java | 12 +-
.../kylin/storage/translate/HBaseKeyRange.java | 2 +-
.../apache/kylin/engine/mr/DFSFileTable.java | 2 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 3 +-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 5 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
.../apache/kylin/engine/spark/SparkCubing.java | 11 +-
.../measure/FixedLenMeasureCodec.java | 3 +-
.../cube/v1/CubeSegmentTopNTupleIterator.java | 86 --------------
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 46 ++------
.../hbase/cube/v1/CubeTupleConverter.java | 60 +---------
.../cube/v1/SerializedHBaseTupleIterator.java | 14 +--
.../observer/ObserverAggregators.java | 20 +++-
.../hbase/cube/v1/filter/FuzzyRowFilterV2.java | 6 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 4 +-
.../storage/hbase/cube/v2/CubeStorageQuery.java | 67 +++--------
.../hbase/cube/v2/CubeTupleConverter.java | 65 +----------
.../hbase/cube/v2/HBaseReadonlyStore.java | 1 -
.../v2/SequentialCubeTopNTupleIterator.java | 68 -----------
.../cube/v2/SequentialCubeTupleIterator.java | 2 +-
.../endpoint/EndpointAggregators.java | 7 +-
.../endpoint/EndpointTupleIterator.java | 3 +-
.../storage/hbase/steps/CreateHTableJob.java | 7 +-
.../storage/hbase/steps/HBaseMROutput2.java | 2 +-
.../hbase/steps/HBaseMROutput2Transition.java | 2 +-
.../storage/hbase/steps/RowValueDecoder.java | 21 +---
.../hbase/util/GridTableHBaseBenchmark.java | 2 +-
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 8 +-
66 files changed, 473 insertions(+), 977 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index d2a94c5..a4b2b1a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -279,14 +279,20 @@ public class KylinConfig implements Serializable {
return getOptional(KYLIN_STORAGE_URL);
}
+ /** was for route to hive, not used any more */
+ @Deprecated
public String getHiveUrl() {
return getOptional(HIVE_URL, "");
}
+ /** was for route to hive, not used any more */
+ @Deprecated
public String getHiveUser() {
return getOptional(HIVE_USER, "");
}
+ /** was for route to hive, not used any more */
+ @Deprecated
public String getHivePassword() {
return getOptional(HIVE_PASSWORD, "");
}
@@ -302,6 +308,10 @@ public class KylinConfig implements Serializable {
public String getHBaseClusterFs() {
return getOptional(KYLIN_HBASE_CLUSTER_FS, "");
}
+
+ public String[] getMeasureTypeFactories() {
+ return getOptionalStringArray("kylin.measure.type.factories", new String[0]);
+ }
public String getKylinJobLogDir() {
return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs");
@@ -445,7 +455,7 @@ public class KylinConfig implements Serializable {
}
public String[] getRestServers() {
- return getOptionalStringArray(KYLIN_REST_SERVERS);
+ return getOptionalStringArray(KYLIN_REST_SERVERS, new String[0]);
}
public String getAdminDls() {
@@ -532,12 +542,12 @@ public class KylinConfig implements Serializable {
return property != null ? property : properties.getProperty(prop);
}
- private String[] getOptionalStringArray(String prop) {
+ private String[] getOptionalStringArray(String prop, String[] dft) {
final String property = getOptional(prop);
if (!StringUtils.isBlank(property)) {
return property.split("\\s*,\\s*");
} else {
- return new String[] {};
+ return dft;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index 4a95c66..065d969 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -108,7 +108,7 @@ public class TopNCounter<T> implements Iterable<Counter<T>> {
incrementCounter(counterNode, incrementCount);
- return new Pair<Boolean, T>(isNewItem, droppedItem);
+ return Pair.newPair(isNewItem, droppedItem);
}
protected void incrementCounter(ListNode2<Counter<T>> counterNode, double incrementCount) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index cc5346c..aa25c22 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -104,7 +104,7 @@ public class CliCommandExecutor {
sshOutput = ssh.execCommand(command, remoteTimeoutSeconds, logAppender);
int exitCode = sshOutput.getExitCode();
String output = sshOutput.getText();
- return new Pair<Integer, String>(exitCode, output);
+ return Pair.newPair(exitCode, output);
} catch (IOException e) {
throw e;
} catch (Exception e) {
@@ -140,7 +140,7 @@ public class CliCommandExecutor {
try {
int exitCode = proc.waitFor();
- return new Pair<Integer, String>(exitCode, result.toString());
+ return Pair.newPair(exitCode, result.toString());
} catch (InterruptedException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Pair.java b/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
index 9efdf3b..d28b05f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
@@ -28,6 +28,7 @@ import java.io.Serializable;
*/
public class Pair<T1, T2> implements Serializable {
private static final long serialVersionUID = -3986244606585552569L;
+
protected T1 first = null;
protected T2 second = null;
@@ -96,7 +97,7 @@ public class Pair<T1, T2> implements Serializable {
}
@Override
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
public boolean equals(Object other) {
return other instanceof Pair && equals(first, ((Pair) other).first) && equals(second, ((Pair) other).second);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterTest.java b/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterTest.java
index a6aa610..821a0fc 100644
--- a/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterTest.java
@@ -248,7 +248,7 @@ public class TopNCounterTest {
List<Pair<String, Double>> allRecords = Lists.newArrayList();
for (Counter<String> counter : tops)
- allRecords.add(new Pair(counter.getItem(), counter.getCount()));
+ allRecords.add(Pair.newPair(counter.getItem(), counter.getCount()));
timeSpent += (System.currentTimeMillis() - startTime);
return allRecords;
}
@@ -284,7 +284,7 @@ public class TopNCounterTest {
List<Pair<String, Double>> allRecords = Lists.newArrayList();
for (Map.Entry<String, Double> entry : hashMap.entrySet()) {
- allRecords.add(new Pair(entry.getKey(), entry.getValue()));
+ allRecords.add(Pair.newPair(entry.getKey(), entry.getValue()));
}
Collections.sort(allRecords, new Comparator<Pair<String, Double>>() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 624bb0b..28252a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -169,11 +169,6 @@ public class CubeCapabilityChecker {
continue;
}
- // calcite can not handle distinct count
- if (functionDesc.isCountDistinct()) {
- continue;
- }
-
// calcite can do aggregation from columns on-the-fly
List<TblColRef> neededCols = functionDesc.getParameter().getColRefs();
if (neededCols.size() > 0 && cubeDesc.listDimensionColumnsIncludingDerived().containsAll(neededCols)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 53cd00f..93bde26 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -34,6 +34,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.slf4j.Logger;
@@ -53,17 +54,13 @@ public class CubeDescManager {
// static cached instances
private static final ConcurrentHashMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
- // ============================================================================
-
- private KylinConfig config;
- // name ==> CubeDesc
- private CaseInsensitiveStringCache<CubeDesc> cubeDescMap;
-
public static CubeDescManager getInstance(KylinConfig config) {
CubeDescManager r = CACHE.get(config);
if (r != null) {
return r;
}
+
+ MeasureTypeFactory.init(config);
synchronized (CubeDescManager.class) {
r = CACHE.get(config);
@@ -87,6 +84,12 @@ public class CubeDescManager {
CACHE.clear();
}
+ // ============================================================================
+
+ private KylinConfig config;
+ // name ==> CubeDesc
+ private CaseInsensitiveStringCache<CubeDesc> cubeDescMap;
+
private CubeDescManager(KylinConfig config) throws IOException {
logger.info("Initializing CubeDescManager with config " + config);
this.config = config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 81c7909..dccc3f1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -421,9 +421,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
return false;
- if (this.getDescriptor().hasHolisticCountDistinctMeasures())
- return false;
-
return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 24286cc..885dafa 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -399,7 +399,7 @@ public class CubeManager implements IRealizationProvider {
CubeUpdate cubeBuilder = new CubeUpdate(cube).setToAddSegs(appendSegment, mergeSegment);
updateCube(cubeBuilder);
- return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
+ return Pair.newPair(appendSegment, mergeSegment);
}
public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException {
@@ -491,7 +491,7 @@ public class CubeManager implements IRealizationProvider {
}
}
}
- return new Pair<Long, Long>(start, end);
+ return Pair.newPair(start, end);
}
private boolean hasOverlap(long startDate, long endDate, long anotherStartDate, long anotherEndDate) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index 7f38c26..d766776 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -125,7 +125,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length];
for (int i = 0; i < result.length; i++) {
int col = columns.trueBitAt(i);
- result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col).toString());
+ result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col));
}
// deal with holistic distinct count
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 361d96a..d0fca02 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -15,8 +15,6 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.collect.LinkedListMultimap;
-import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -32,7 +30,7 @@ public class CuboidToGridTableMapping {
private ImmutableBitSet gtPrimaryKey;
private int nMetrics;
- private ListMultimap<FunctionDesc, Integer> metrics2gt; // because count distinct may have a holistic version
+ private Map<FunctionDesc, Integer> metrics2gt; // because count distinct may have a holistic version
public CuboidToGridTableMapping(Cuboid cuboid) {
this.cuboid = cuboid;
@@ -68,19 +66,12 @@ public class CuboidToGridTableMapping {
}
// metrics
- metrics2gt = LinkedListMultimap.create();
+ metrics2gt = Maps.newHashMap();
for (MeasureDesc measure :cuboid.getCubeDesc().getMeasures()) {
// Count distinct & holistic count distinct are equals() but different.
// Ensure the holistic version if exists is always the first.
FunctionDesc func = measure.getFunction();
- if (func.isHolisticCountDistinct()) {
- List<Integer> existing = metrics2gt.removeAll(func);
- metrics2gt.put(func, gtColIdx);
- metrics2gt.putAll(func, existing);
- } else {
- metrics2gt.put(func, gtColIdx);
- }
-
+ metrics2gt.put(func, gtColIdx);
gtDataTypes.add(func.getReturnDataType());
// map to column block
@@ -135,19 +126,8 @@ public class CuboidToGridTableMapping {
}
public int getIndexOf(FunctionDesc metric) {
- List<Integer> list = metrics2gt.get(metric);
- // normal case
- if (list.size() == 1) {
- return list.get(0);
- }
- // count distinct & its holistic version
- else if (list.size() == 2) {
- assert metric.isCountDistinct();
- return metric.isHolisticCountDistinct() ? list.get(0) : list.get(1);
- }
- // unexpected
- else
- return -1;
+ Integer r = metrics2gt.get(metric);
+ return r == null ? -1 : r.intValue();
}
public List<TblColRef> getCuboidDimensionsInGTOrder() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index c4d0a7e..19ebaef 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -94,7 +94,7 @@ public class TrimmedCubeCodeSystem implements IGTCodeSystem {
MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length];
for (int i = 0; i < result.length; i++) {
int col = columns.trueBitAt(i);
- result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col).toString());
+ result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col));
}
// deal with holistic distinct count
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index fed9479..3e9a67d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -99,8 +99,7 @@ public class InMemCubeBuilderInputConverter {
int paramColIdx = 0; // index among parameters of column type
for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
String value;
- if (function.isCount() || function.isHolisticCountDistinct()) {
- // note for holistic count distinct, this value will be ignored
+ if (function.isCount()) {
value = "1";
} else if (param.isColumnType()) {
value = row.get(colIdxOnFlatTable[paramColIdx++]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index f8a887d..8fa99cd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -17,43 +17,15 @@
*/
package org.apache.kylin.cube.inmemcubing;
-import com.google.common.collect.Maps;
+import java.util.BitSet;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.Map;
/**
*/
public final class InMemCubeBuilderUtils {
- public static final HashMap<Integer, Dictionary<String>> createTopNLiteralColDictionaryMap(CubeDesc cubeDesc, CubeJoinedFlatTableDesc intermediateTableDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- HashMap<Integer, Dictionary<String>> result = Maps.newHashMap();
- for (int measureIdx = 0; measureIdx < cubeDesc.getMeasures().size(); measureIdx++) {
- MeasureDesc measureDesc = cubeDesc.getMeasures().get(measureIdx);
- FunctionDesc func = measureDesc.getFunction();
- if (func.isTopN()) {
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];
- int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef literalCol = func.getTopNLiteralColumn();
- @SuppressWarnings("unchecked")
- Dictionary<String> dictionary = (Dictionary<String>) dictionaryMap.get(literalCol);
- //Preconditions.checkNotNull(dictionary);//FIXME disable check since dictionary is null when building empty segment
- result.put(literalColIdx, dictionary);
- }
- }
- return result;
- }
-
public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
int cardinality = Long.bitCount(cuboidId);
BitSet dimension = new BitSet();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 8c7d891..7e79f64 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -642,11 +642,6 @@ public class CubeDesc extends RootPersistentEntity {
FunctionDesc func = m.getFunction();
func.init(factTable);
allColumns.addAll(func.getParameter().getColRefs());
-
- // verify holistic count distinct as a dependent measure
- if (func.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
- throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
- }
}
}
@@ -677,9 +672,9 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- public boolean hasHolisticCountDistinctMeasures() {
+ public boolean hasMemoryHungryMeasures() {
for (MeasureDesc measure : measures) {
- if (measure.getFunction().isHolisticCountDistinct()) {
+ if (measure.getFunction().getMeasureType().isMemoryHungry()) {
return true;
}
}
@@ -766,7 +761,7 @@ public class CubeDesc extends RootPersistentEntity {
}
for (MeasureDesc measure : measures) {
- MeasureType aggrType = measure.getFunction().getMeasureType();
+ MeasureType<?> aggrType = measure.getFunction().getMeasureType();
result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
}
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index c918798..530ce07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -114,10 +114,6 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
if (rtype.isIntegerFamily() == false) {
context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.INTEGER_FAMILY);
}
- } else if (funcDesc.isCountDistinct()) {
- if (rtype.isHLLC() == false && funcDesc.isHolisticCountDistinct() == false) {
- context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be hllc(10), hllc(12) etc.");
- }
} else if (funcDesc.isMax() || funcDesc.isMin() || funcDesc.isSum()) {
if (rtype.isNumberFamily() == false) {
context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index b3133be..77eb430 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -60,7 +60,7 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length];
for (int i = 0; i < result.length; i++) {
int col = columns.trueBitAt(i);
- result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col).toString());
+ result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index ab87c2b..b593e48 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -186,15 +186,22 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
for (int measureIdx = 0; measureIdx < cube.getDescriptor().getMeasures().size(); measureIdx++) {
MeasureDesc measureDesc = cube.getDescriptor().getMeasures().get(measureIdx);
FunctionDesc func = measureDesc.getFunction();
- if (func.isTopN()) {
- int[] flatTableIdx = flatTableDesc.getMeasureColumnIndexes()[measureIdx];
- int literalColIdx = flatTableIdx[flatTableIdx.length - 1];
- TblColRef literalCol = func.getTopNLiteralColumn();
- logger.info("Building dictionary for " + literalCol);
- List<byte[]> valueList = readValueList(flatTable, nColumns, literalColIdx);
- Dictionary<String> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(literalCol.getType(), new IterableDictionaryValueEnumerator(valueList));
-
- result.put(literalCol, dict);
+ List<TblColRef> dictCols = func.getMeasureType().getColumnsNeedDictionary(func);
+ if (dictCols.isEmpty())
+ continue;
+
+ int[] flatTableIdx = flatTableDesc.getMeasureColumnIndexes()[measureIdx];
+ List<TblColRef> paramCols = func.getParameter().getColRefs();
+ for (int i = 0; i < paramCols.size(); i++) {
+ TblColRef col = paramCols.get(i);
+ if (dictCols.contains(col)) {
+ int colIdxOnFlat = flatTableIdx[i];
+ logger.info("Building dictionary for " + col);
+ List<byte[]> valueList = readValueList(flatTable, nColumns, colIdxOnFlat);
+ Dictionary<String> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(col.getType(), new IterableDictionaryValueEnumerator(valueList));
+
+ result.put(col, dict);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index eb2b963..9abd28c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -125,7 +125,7 @@ abstract public class LookupTable<T extends Comparable<T>> {
if (returnBegin == null && returnEnd == null)
return null;
else
- return new Pair<T, T>(returnBegin, returnEnd);
+ return Pair.newPair(returnBegin, returnEnd);
}
public Set<T> mapValues(String col, Set<T> values, String returnCol) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
deleted file mode 100644
index 0ab547a..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure;
-
-public interface IMeasureFactory {
-
- public MeasureType createMeasureType(String funcName, String dataType);
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
index 32e5128..4c031c7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
@@ -20,13 +20,15 @@ package org.apache.kylin.measure;
import java.io.Serializable;
+import org.apache.kylin.metadata.datatype.DataType;
+
/**
*/
@SuppressWarnings("serial")
abstract public class MeasureAggregator<V> implements Serializable {
- public static MeasureAggregator<?> create(String funcName, String dataType) {
- return MeasureType.create(funcName, dataType).newAggregator();
+ public static MeasureAggregator<?> create(String funcName, DataType dataType) {
+ return MeasureTypeFactory.create(funcName, dataType).newAggregator();
}
public static int guessBigDecimalMemBytes() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index c6b456e..b3edbc3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -45,7 +45,7 @@ public class MeasureAggregators implements Serializable {
Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
for (int i = 0; i < descLength; i++) {
FunctionDesc func = measureDescs[i].getFunction();
- aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
+ aggs[i] = func.getMeasureType().newAggregator();
measureIndexMap.put(measureDescs[i].getName(), i);
}
// fill back dependent aggregator
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index d4b2700..cf04c60 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -24,11 +24,6 @@ import java.util.List;
import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.measure.basic.BasicMeasureFactory;
-import org.apache.kylin.measure.hllc.HLLCAggregationFactory;
-import org.apache.kylin.measure.topn.TopNMeasureFactory;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -37,54 +32,27 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
-import com.google.common.collect.Maps;
-
-abstract public class MeasureType {
-
- private static final Map<String, IMeasureFactory> factoryRegistry = Maps.newConcurrentMap();
- private static final IMeasureFactory defaultFactory = new BasicMeasureFactory();
-
- static {
- factoryRegistry.put(FunctionDesc.FUNC_COUNT_DISTINCT, new HLLCAggregationFactory());
- factoryRegistry.put(FunctionDesc.FUNC_TOP_N, new TopNMeasureFactory());
- }
-
- public static MeasureType create(String funcName, String dataType) {
- funcName = funcName.toUpperCase();
- dataType = dataType.toLowerCase();
-
- IMeasureFactory factory = factoryRegistry.get(funcName);
- if (factory == null)
- factory = defaultFactory;
-
- MeasureType result = factory.createMeasureType(funcName, dataType);
-
- // register serializer for aggr data type
- DataType aggregationDataType = result.getAggregationDataType();
- if (DataTypeSerializer.hasRegistered(aggregationDataType.getName()) == false) {
- DataTypeSerializer.register(aggregationDataType.getName(), result.getAggregationDataSeralizer());
- }
-
- return result;
- }
+abstract public class MeasureType<T> {
/* ============================================================================
* Define
* ---------------------------------------------------------------------------- */
- abstract public DataType getAggregationDataType();
-
- abstract public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer();
+ public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
+ return;
+ }
- abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
+ public boolean isMemoryHungry() {
+ return false;
+ }
/* ============================================================================
* Build
* ---------------------------------------------------------------------------- */
- abstract public MeasureIngester<?> newIngester();
+ abstract public MeasureIngester<T> newIngester();
- abstract public MeasureAggregator<?> newAggregator();
+ abstract public MeasureAggregator<T> newAggregator();
public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
return Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
new file mode 100644
index 0000000..48eddb2
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.measure.hllc.HLLCMeasureType;
+import org.apache.kylin.measure.topn.TopNMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+abstract public class MeasureTypeFactory<T> {
+
+ abstract public MeasureType<T> createMeasureType(String funcName, DataType dataType);
+
+ abstract public String getAggrFunctionName();
+ abstract public String getAggrDataTypeName();
+ abstract public Class<? extends DataTypeSerializer<T>> getAggrDataTypeSerializer();
+
+ // ============================================================================
+
+ private static Map<Pair<String, String>, MeasureTypeFactory<?>> factories = Maps.newHashMap();
+ private static MeasureTypeFactory<?> defaultFactory = new BasicMeasureType.Factory();
+
+ public static synchronized void init(KylinConfig config) {
+ if (factories.isEmpty() == false)
+ return;
+
+ List<MeasureTypeFactory<?>> factoryInsts = Lists.newArrayList();
+
+ // two built-in advanced measure types
+ factoryInsts.add(new HLLCMeasureType.Factory());
+ factoryInsts.add(new TopNMeasureType.Factory());
+
+ // more custom measure types
+ for (String factoryClz : config.getMeasureTypeFactories()) {
+ factoryInsts.add((MeasureTypeFactory<?>) ClassUtil.newInstance(factoryClz));
+ }
+
+ // register factories & data type serializers
+ for (MeasureTypeFactory<?> factory : factoryInsts) {
+ String funcName = factory.getAggrFunctionName().toUpperCase();
+ String dataTypeName = factory.getAggrDataTypeName().toLowerCase();
+ Class<? extends DataTypeSerializer<?>> serializer = factory.getAggrDataTypeSerializer();
+
+ DataType.register(dataTypeName);
+ DataTypeSerializer.register(dataTypeName, serializer);
+ factories.put(Pair.newPair(funcName, dataTypeName), factory);
+ }
+ }
+
+ public static MeasureType<?> create(String funcName, String dataType) {
+ return create(funcName, DataType.getType(dataType));
+ }
+
+ public static MeasureType<?> create(String funcName, DataType dataType) {
+ funcName = funcName.toUpperCase();
+
+ MeasureTypeFactory<?> factory = factories.get(Pair.newPair(funcName, dataType.getName()));
+ if (factory == null)
+ factory = defaultFactory;
+
+ return factory.createMeasureType(funcName, dataType);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
deleted file mode 100644
index 7bfee49..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.basic;
-
-import org.apache.kylin.measure.IMeasureFactory;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
-public class BasicMeasureFactory implements IMeasureFactory {
-
- @Override
- public MeasureType createMeasureType(String funcName, String dataType) {
- return new BasicMeasureType(funcName, DataType.getType(dataType));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index fea6ba5..bc9e6b1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -21,47 +21,59 @@ package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.BigDecimalSerializer;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.apache.kylin.metadata.datatype.DoubleSerializer;
-import org.apache.kylin.metadata.datatype.LongSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
+@SuppressWarnings("rawtypes")
public class BasicMeasureType extends MeasureType {
+ public static class Factory extends MeasureTypeFactory {
+
+ @Override
+ public MeasureType createMeasureType(String funcName, DataType dataType) {
+ return new BasicMeasureType(funcName, dataType);
+ }
+
+ @Override
+ public String getAggrFunctionName() {
+ return null;
+ }
+
+ @Override
+ public String getAggrDataTypeName() {
+ return null;
+ }
+
+ @Override
+ public Class getAggrDataTypeSerializer() {
+ return null;
+ }
+ }
+
private final String funcName;
private final DataType dataType;
public BasicMeasureType(String funcName, DataType dataType) {
+ validate(funcName, dataType);
this.funcName = funcName;
this.dataType = dataType;
}
@Override
- public DataType getAggregationDataType() {
- return dataType;
+ public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
+ validate(functionDesc.getExpression(), functionDesc.getReturnDataType());
}
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- if (dataType.isIntegerFamily())
- return LongSerializer.class;
- else if (dataType.isDecimal())
- return BigDecimalSerializer.class;
- else if (dataType.isNumberFamily())
- return DoubleSerializer.class;
- else
- throw new IllegalArgumentException("No serializer for aggregation type " + dataType);
+ private void validate(String funcName, DataType dataType) throws IllegalArgumentException {
+ if ((funcName.equals(FunctionDesc.FUNC_SUM) //
+ || funcName.equals(FunctionDesc.FUNC_COUNT) //
+ || funcName.equals(FunctionDesc.FUNC_MAX) //
+ || funcName.equals(FunctionDesc.FUNC_MIN)) == false)
+ throw new IllegalArgumentException();
}
@Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
public MeasureIngester<?> newIngester() {
if (dataType.isIntegerFamily())
return new LongIngester();
@@ -72,7 +84,7 @@ public class BasicMeasureType extends MeasureType {
else
throw new IllegalArgumentException("No ingester for aggregation type " + dataType);
}
-
+
@Override
public MeasureAggregator<?> newAggregator() {
if (isSum() || isCount()) {
@@ -99,21 +111,21 @@ public class BasicMeasureType extends MeasureType {
}
throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + dataType + "'");
}
-
+
private boolean isSum() {
- return FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName);
+ return FunctionDesc.FUNC_SUM.equals(funcName);
}
private boolean isCount() {
- return FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName);
+ return FunctionDesc.FUNC_COUNT.equals(funcName);
}
-
+
private boolean isMax() {
- return FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName);
+ return FunctionDesc.FUNC_MAX.equals(funcName);
}
-
+
private boolean isMin() {
- return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
+ return FunctionDesc.FUNC_MIN.equals(funcName);
}
@Override
@@ -126,5 +138,5 @@ public class BasicMeasureType extends MeasureType {
// TODO Auto-generated method stub
return null;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
deleted file mode 100644
index 13e5520..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hllc;
-
-import org.apache.kylin.measure.IMeasureFactory;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-public class HLLCAggregationFactory implements IMeasureFactory {
-
- @Override
- public MeasureType createMeasureType(String funcName, String dataType) {
- if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName) == false)
- throw new IllegalArgumentException();
-
- return new HLLCMeasureType(DataType.getType(dataType));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 83fcf3b..073de6d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -25,43 +25,72 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class HLLCMeasureType extends MeasureType {
+public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
+
+ public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
+ public static final String DATATYPE_HLLC = "hllc";
+
+ public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounter> {
+
+ @Override
+ public MeasureType<HyperLogLogPlusCounter> createMeasureType(String funcName, DataType dataType) {
+ return new HLLCMeasureType(funcName, dataType);
+ }
+
+ @Override
+ public String getAggrFunctionName() {
+ return FUNC_COUNT_DISTINCT;
+ }
+
+ @Override
+ public String getAggrDataTypeName() {
+ return DATATYPE_HLLC;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<HyperLogLogPlusCounter>> getAggrDataTypeSerializer() {
+ return HLLCSerializer.class;
+ }
+ }
+
+ // ============================================================================
private final DataType dataType;
- public HLLCMeasureType(DataType dataType) {
- if ("hllc".equals(dataType.getName()) == false)
- throw new IllegalArgumentException();
-
+ public HLLCMeasureType(String funcName, DataType dataType) {
+ validate(funcName, dataType);
this.dataType = dataType;
-
- if (this.dataType.getPrecision() < 10 || this.dataType.getPrecision() > 16)
- throw new IllegalArgumentException("HLLC precision must be between 10 and 16");
}
- @Override
- public DataType getAggregationDataType() {
- return dataType;
+ public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
+ validate(functionDesc.getExpression(), functionDesc.getReturnDataType());
}
- @Override
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- return HLLCSerializer.class;
+ private void validate(String funcName, DataType dataType) {
+ if (FUNC_COUNT_DISTINCT.equals(funcName) == false)
+ throw new IllegalArgumentException();
+
+ if (DATATYPE_HLLC.equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+
+ if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
+ throw new IllegalArgumentException();
}
@Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
-
+ public boolean isMemoryHungry() {
+ return true;
}
@Override
- public MeasureIngester<?> newIngester() {
+ public MeasureIngester<HyperLogLogPlusCounter> newIngester() {
return new MeasureIngester<HyperLogLogPlusCounter>() {
HyperLogLogPlusCounter current = new HyperLogLogPlusCounter(dataType.getPrecision());
@@ -77,11 +106,8 @@ public class HLLCMeasureType extends MeasureType {
}
@Override
- public MeasureAggregator<?> newAggregator() {
- if (dataType.isHLLC())
- return new HLLCAggregator(dataType.getPrecision());
- else
- return new LDCAggregator();
+ public MeasureAggregator<HyperLogLogPlusCounter> newAggregator() {
+ return new HLLCAggregator(dataType.getPrecision());
}
@Override
@@ -95,4 +121,8 @@ public class HLLCMeasureType extends MeasureType {
return null;
}
+ public static boolean isCountDistinct(FunctionDesc func) {
+ return FUNC_COUNT_DISTINCT.equalsIgnoreCase(func.getExpression());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index 7131201..7424493 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -81,7 +81,8 @@ public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
@Override
public int getStorageBytesEstimate() {
- return current().maxLength();
+ // for HLL, it will be compressed when export to bytes
+ return (int) (current().maxLength() * 0.75);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
deleted file mode 100644
index 5d96450..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/LDCAggregator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hllc;
-
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
-
-/**
- * Long Distinct Count
- */
-@SuppressWarnings("serial")
-public class LDCAggregator extends MeasureAggregator<LongMutable> {
-
- private static LongMutable ZERO = new LongMutable(0);
-
- private HLLCAggregator hllAgg = null;
- private LongMutable state = new LongMutable(0);
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- this.hllAgg = (HLLCAggregator) agg;
- }
-
- @Override
- public void reset() {
- }
-
- @Override
- public void aggregate(LongMutable value) {
- }
-
- @Override
- public LongMutable getState() {
- if (hllAgg == null) {
- return ZERO;
- } else {
- state.set(hllAgg.getState().getCountEstimate());
- return state;
- }
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
deleted file mode 100644
index 1ceb607..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.topn;
-
-import org.apache.kylin.measure.IMeasureFactory;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-public class TopNMeasureFactory implements IMeasureFactory {
-
- @Override
- public MeasureType createMeasureType(String funcName, String dataType) {
- if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName) == false)
- throw new IllegalArgumentException();
-
- return new TopNMeasureType(DataType.getType(dataType));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 01582d0..c3d36ca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -32,7 +32,7 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.hllc.HLLCSerializer;
+import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -47,44 +47,70 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
-public class TopNMeasureType extends MeasureType {
+public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
- private final DataType dataType;
+ public static final String FUNC_TOP_N = "TOP_N";
+ public static final String DATATYPE_TOPN = "topn";
+
+ public static class Factory extends MeasureTypeFactory<TopNCounter<ByteArray>> {
- public TopNMeasureType(DataType dataType) {
- if ("topn".equals(dataType.getName()) == false)
- throw new IllegalArgumentException();
+ @Override
+ public MeasureType<TopNCounter<ByteArray>> createMeasureType(String funcName, DataType dataType) {
+ return new TopNMeasureType(funcName, dataType);
+ }
- this.dataType = dataType;
+ @Override
+ public String getAggrFunctionName() {
+ return FUNC_TOP_N;
+ }
+
+ @Override
+ public String getAggrDataTypeName() {
+ return DATATYPE_TOPN;
+ }
- if (this.dataType.getPrecision() < 1 || this.dataType.getPrecision() > 1000)
- throw new IllegalArgumentException("TopN precision must be between 1 and 1000");
+ @Override
+ public Class<? extends DataTypeSerializer<TopNCounter<ByteArray>>> getAggrDataTypeSerializer() {
+ return TopNCounterSerializer.class;
+ }
}
+
+ // ============================================================================
- @Override
- public DataType getAggregationDataType() {
- return dataType;
+ private final DataType dataType;
+
+ public TopNMeasureType(String funcName, DataType dataType) {
+ validate(funcName, dataType);
+ this.dataType = dataType;
}
- @Override
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- return HLLCSerializer.class;
+ public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
+ validate(functionDesc.getExpression(), functionDesc.getReturnDataType());
}
- @Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
+ private void validate(String funcName, DataType dataType) {
+ if (FUNC_TOP_N.equals(funcName) == false)
+ throw new IllegalArgumentException();
+
+ if (DATATYPE_TOPN.equals(dataType.getName()) == false)
+ throw new IllegalArgumentException();
+ if (dataType.getPrecision() < 1 || dataType.getPrecision() > 5000)
+ throw new IllegalArgumentException();
}
- @SuppressWarnings("rawtypes")
@Override
- public MeasureIngester<?> newIngester() {
- return new MeasureIngester<TopNCounter>() {
+ public boolean isMemoryHungry() {
+ return true;
+ }
+
+ @Override
+ public MeasureIngester<TopNCounter<ByteArray>> newIngester() {
+ return new MeasureIngester<TopNCounter<ByteArray>>() {
@Override
- public TopNCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length != 2)
throw new IllegalArgumentException();
@@ -92,7 +118,7 @@ public class TopNMeasureType extends MeasureType {
String literal = values[1];
// encode literal using dictionary
- TblColRef literalCol = measureDesc.getFunction().getTopNLiteralColumn();
+ TblColRef literalCol = getTopNLiteralColumn(measureDesc.getFunction());
Dictionary<String> dictionary = dictionaryMap.get(literalCol);
int keyEncodedValue = dictionary.getIdFromValue(literal);
@@ -104,12 +130,11 @@ public class TopNMeasureType extends MeasureType {
return topNCounter;
}
- @SuppressWarnings("unchecked")
@Override
- public TopNCounter reEncodeDictionary(TopNCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
- TopNCounter<ByteArray> topNCounter = (TopNCounter<ByteArray>) value;
+ public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
+ TopNCounter<ByteArray> topNCounter = value;
- TblColRef colRef = measureDesc.getFunction().getTopNLiteralColumn();
+ TblColRef colRef = getTopNLiteralColumn(measureDesc.getFunction());
Dictionary<String> sourceDict = oldDicts.get(colRef);
Dictionary<String> mergedDict = newDicts.get(colRef);
@@ -138,7 +163,7 @@ public class TopNMeasureType extends MeasureType {
}
@Override
- public MeasureAggregator<?> newAggregator() {
+ public MeasureAggregator<TopNCounter<ByteArray>> newAggregator() {
return new TopNAggregator();
}
@@ -159,11 +184,11 @@ public class TopNMeasureType extends MeasureType {
// the measure function must be SUM
FunctionDesc onlyFunction = digest.aggregations.iterator().next();
- if (onlyFunction.isSum() == false)
+ if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
return null;
- TblColRef literalCol = topN.getFunction().getTopNLiteralColumn();
- if (unmatchedDimensions.contains(literalCol) && topN.getFunction().isTopNCompatibleSum(onlyFunction)) {
+ TblColRef literalCol = getTopNLiteralColumn(topN.getFunction());
+ if (unmatchedDimensions.contains(literalCol)) {
unmatchedDimensions.remove(literalCol);
unmatchedAggregations.remove(onlyFunction);
return new CapabilityInfluence() {
@@ -176,6 +201,21 @@ public class TopNMeasureType extends MeasureType {
return null;
}
+ private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) {
+ if (sum == null)
+ return false;
+
+ if (!isTopN(topN) || !sum.isSum())
+ return false;
+
+ if (sum.getParameter().getColRefs().isEmpty())
+ return false;
+
+ TblColRef sumCol = sum.getParameter().getColRefs().get(0);
+ TblColRef topnNumCol = getTopNNumericColumn(topN);
+ return sumCol.equals(topnNumCol);
+ }
+
@Override
public boolean needRewrite() {
return false;
@@ -189,7 +229,7 @@ public class TopNMeasureType extends MeasureType {
@Override
public void beforeStorageQuery(MeasureDesc measureDesc, SQLDigest sqlDigest) {
FunctionDesc topnFunc = measureDesc.getFunction();
- TblColRef topnLiteralCol = topnFunc.getTopNLiteralColumn();
+ TblColRef topnLiteralCol = getTopNLiteralColumn(topnFunc);
if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false)
return;
@@ -213,12 +253,12 @@ public class TopNMeasureType extends MeasureType {
public boolean needAdvancedTupleFilling() {
return true;
}
-
+
@Override
public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
throw new UnsupportedOperationException();
}
-
+
@Override
public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
final TblColRef literalCol = getTopNLiteralColumn(function);
@@ -227,7 +267,7 @@ public class TopNMeasureType extends MeasureType {
final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;
// for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite)
final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1;
-
+
return new IAdvMeasureFiller() {
private TopNCounter<ByteArray> topNCounter;
private Iterator<Counter<ByteArray>> topNCounterIterator;
@@ -249,7 +289,7 @@ public class TopNMeasureType extends MeasureType {
public void fillTuplle(Tuple tuple, int row) {
if (expectRow++ != row)
throw new IllegalStateException();
-
+
Counter<ByteArray> counter = topNCounterIterator.next();
int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
String colValue = topNColDict.getValueFromId(key);
@@ -266,4 +306,8 @@ public class TopNMeasureType extends MeasureType {
private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) {
return functionDesc.getParameter().getColRefs().get(1);
}
+
+ private boolean isTopN(FunctionDesc functionDesc) {
+ return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 235c99f..05e6744 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.datatype;
import java.io.Serializable;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,21 +29,37 @@ import java.util.concurrent.ConcurrentMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+
/**
*/
@SuppressWarnings("serial")
public class DataType implements Serializable {
- // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
- public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
- + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
- + "|date|time|datetime|timestamp|byte|int|short|long|string|hllc|topn";
+ private static final LinkedHashSet<String> VALID_TYPES = new LinkedHashSet<String>();
+ private static Pattern TYPE_PATTERN = null;
private static final String TYPE_PATTEN_TAIL = "\\s*" //
+ "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?";
- private static final Pattern TYPE_PATTERN = Pattern.compile( //
- "(" + VALID_TYPES_STRING + ")" + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
+ public static synchronized void register(String... typeNames) {
+ for (String typeName : typeNames) {
+ VALID_TYPES.add(typeName);
+ }
+
+ TYPE_PATTERN = Pattern.compile( //
+ "(" + StringUtils.join(VALID_TYPES, "|") + ")" //
+ + TYPE_PATTEN_TAIL, Pattern.CASE_INSENSITIVE);
+ }
+
+ // standard sql types, ref: http://www.w3schools.com/sql/sql_datatypes_general.asp
+ static {
+ register("any", "char", "varchar", "string", //
+ "boolean", "byte", "binary", //
+ "int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
+ "float", "real", "double", "decimal", "numeric", //
+ "date", "time", "datetime", "timestamp");
+ }
public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
@@ -219,10 +236,6 @@ public class DataType implements Serializable {
return name.equals("decimal");
}
- public boolean isHLLC() {
- return name.equals("hllc");
- }
-
public String getName() {
return name;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index fd3121f..01f8cc6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -19,13 +19,9 @@
package org.apache.kylin.metadata.datatype;
import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.measure.hllc.HLLCSerializer;
-import org.apache.kylin.measure.topn.TopNCounterSerializer;
import com.google.common.collect.Maps;
@@ -36,28 +32,20 @@ import com.google.common.collect.Maps;
*/
abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
- final static Map<String, Class<?>> implementations;
+ final static Map<String, Class<?>> implementations = Maps.newHashMap();
static {
- HashMap<String, Class<?>> impl = Maps.newHashMap();
- impl.put("varchar", StringSerializer.class);
- impl.put("decimal", BigDecimalSerializer.class);
- impl.put("double", DoubleSerializer.class);
- impl.put("float", DoubleSerializer.class);
- impl.put("bigint", LongSerializer.class);
- impl.put("long", LongSerializer.class);
- impl.put("integer", LongSerializer.class);
- impl.put("int", LongSerializer.class);
- impl.put("smallint", LongSerializer.class);
- impl.put("date", DateTimeSerializer.class);
- impl.put("datetime", DateTimeSerializer.class);
- impl.put("timestamp", DateTimeSerializer.class);
- impl.put("topn", TopNCounterSerializer.class);
- impl.put("hllc", HLLCSerializer.class);
- implementations = Collections.unmodifiableMap(impl);
- }
-
- public static boolean hasRegistered(String dataTypeName) {
- return implementations.containsKey(dataTypeName);
+ implementations.put("varchar", StringSerializer.class);
+ implementations.put("decimal", BigDecimalSerializer.class);
+ implementations.put("double", DoubleSerializer.class);
+ implementations.put("float", DoubleSerializer.class);
+ implementations.put("bigint", LongSerializer.class);
+ implementations.put("long", LongSerializer.class);
+ implementations.put("integer", LongSerializer.class);
+ implementations.put("int", LongSerializer.class);
+ implementations.put("smallint", LongSerializer.class);
+ implementations.put("date", DateTimeSerializer.class);
+ implementations.put("datetime", DateTimeSerializer.class);
+ implementations.put("timestamp", DateTimeSerializer.class);
}
public static void register(String dataTypeName, Class<? extends DataTypeSerializer<?>> impl) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 104f3c7..39fe6b3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
import java.util.Collection;
import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -38,8 +40,6 @@ public class FunctionDesc {
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
public static final String FUNC_COUNT = "COUNT";
- public static final String FUNC_COUNT_DISTINCT = "COUNT_DISTINCT";
- public static final String FUNC_TOP_N = "TOP_N";
public static final String PARAMETER_TYPE_CONSTANT = "constant";
public static final String PARAMETER_TYPE_COLUMN = "column";
@@ -52,7 +52,7 @@ public class FunctionDesc {
private String returnType;
private DataType returnDataType;
- private MeasureType measureType;
+ private MeasureType<?> measureType;
private boolean isDimensionAsMetric = false;
public void init(TableDesc factTable) {
@@ -80,12 +80,12 @@ public class FunctionDesc {
}
}
- public MeasureType getMeasureType() {
+ public MeasureType<?> getMeasureType() {
if (isDimensionAsMetric)
return null;
if (measureType == null) {
- measureType = MeasureType.create(getExpression(), getReturnType());
+ measureType = MeasureTypeFactory.create(getExpression(), getReturnDataType());
}
return measureType;
}
@@ -108,12 +108,12 @@ public class FunctionDesc {
}
public DataType getRewriteFieldType() {
- if (isCountDistinct() || isTopN())
- return DataType.ANY;
- else if (isSum() || isMax() || isMin())
+ if (isSum() || isMax() || isMin())
return parameter.getColRefs().get(0).getType();
- else
+ else if (getMeasureType() instanceof BasicMeasureType)
return returnDataType;
+ else
+ return DataType.ANY;
}
public ColumnDesc newFakeRewriteColumn(TableDesc sourceTable) {
@@ -142,22 +142,6 @@ public class FunctionDesc {
return FUNC_COUNT.equalsIgnoreCase(expression);
}
- public boolean isCountDistinct() {
- return FUNC_COUNT_DISTINCT.equalsIgnoreCase(expression);
- }
-
- public boolean isTopN() {
- return FUNC_TOP_N.equalsIgnoreCase(expression);
- }
-
- public boolean isHolisticCountDistinct() {
- if (isCountDistinct() && returnDataType != null && returnDataType.isBigInt()) {
- return true;
- } else {
- return false;
- }
- }
-
/**
* Get Full Expression such as sum(amount), count(1), count(*)...
*/
@@ -272,35 +256,4 @@ public class FunctionDesc {
return "FunctionDesc [expression=" + expression + ", parameter=" + parameter + ", returnType=" + returnType + "]";
}
- // cols[0] is numeric (e.g. GMV), cols[1] is literal (e.g. SELLER)
- public TblColRef getTopNNumericColumn() {
- if (isTopN() == false)
- throw new IllegalStateException();
-
- return parameter.getColRefs().get(0);
- }
-
- // cols[0] is numeric (e.g. GMV), cols[1] is literal (e.g. SELLER)
- public TblColRef getTopNLiteralColumn() {
- if (isTopN() == false)
- throw new IllegalStateException();
-
- return parameter.getColRefs().get(1);
- }
-
- public boolean isTopNCompatibleSum(FunctionDesc sum) {
- if (isTopN() == false)
- throw new IllegalStateException();
-
- if (sum == null) {
- return false;
- }
-
- if (this.isTopN() && sum.isSum()) {
- if (this.getParameter().getColRefs().get(0).equals(sum.getParameter().getColRefs().get(0)))
- return true;
- }
-
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
index 49328e9..c12f18a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TeeTupleIterator.java
@@ -2,9 +2,6 @@ package org.apache.kylin.metadata.tuple;
import java.util.List;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
/**
@@ -13,8 +10,6 @@ import com.google.common.collect.Lists;
*/
public class TeeTupleIterator implements ITupleIterator {
- private static final Logger logger = LoggerFactory.getLogger(TeeTupleIterator.class);
-
private ITupleIterator underlying;
private List<ITuple> duplicatedData;
private List<TeeTupleItrListener> listeners = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7f50c17a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index c79e1f8..d38aafd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -24,8 +24,6 @@ import java.util.List;
import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -61,15 +59,7 @@ public class Tuple implements ITuple {
public ITuple makeCopy() {
Tuple ret = new Tuple(this.info);
for (int i = 0; i < this.values.length; ++i) {
- if (this.values[i] == null) {
- ret.values[i] = null;
- } else if (this.values[i] instanceof HyperLogLogPlusCounter) {
- ret.values[i] = new HyperLogLogPlusCounter((HyperLogLogPlusCounter) this.values[i]);
- } else if (this.values[i] instanceof TopNCounter) {
- ret.values[i] = null;
- } else {
- ret.values[i] = this.values[i];
- }
+ ret.values[i] = this.values[i];
}
return ret;
}
[11/21] kylin git commit: KYLIN-976 Add ingester;
Build part done, in-mem cube test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
deleted file mode 100644
index 7430c4e..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleMinAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DoubleMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
deleted file mode 100644
index 6e66c1b..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/DoubleSumAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DoubleMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable sum = new DoubleMutable();
-
- @Override
- public void reset() {
- sum.set(0.0);
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
deleted file mode 100644
index 7fdf3d8..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMaxAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.LongMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
deleted file mode 100644
index 22ae865..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongMinAggregator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.LongMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
deleted file mode 100644
index 38d728a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/basic/LongSumAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.basic;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.LongMutable;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable sum = new LongMutable();
-
- @Override
- public void reset() {
- sum.set(0);
- }
-
- @Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public LongMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
deleted file mode 100644
index d5ceba5..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregation.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.hllc;
-
-import java.util.List;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public class HLLCAggregation extends AggregationType {
-
- private final DataType dataType;
-
- public HLLCAggregation(String dataType) {
- this.dataType = DataType.getType(dataType);
-
- if (this.dataType.getPrecision() < 10 || this.dataType.getPrecision() > 16)
- throw new IllegalArgumentException("HLLC precision must be between 10 and 16");
- }
-
- @Override
- public DataType getAggregationDataType() {
- return dataType;
- }
-
- @Override
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- return HLLCSerializer.class;
- }
-
- @Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public MeasureAggregator<?> newAggregator() {
- if (dataType.isHLLC())
- return new HLLCAggregator(dataType.getPrecision());
- else
- return new LDCAggregator();
- }
-
- @Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
deleted file mode 100644
index 18c021d..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregationFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.hllc;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.IAggregationFactory;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-public class HLLCAggregationFactory implements IAggregationFactory {
-
- @Override
- public AggregationType createAggregationType(String funcName, String dataType) {
- if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName) == false)
- throw new IllegalArgumentException();
-
- return new HLLCAggregation(dataType);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
deleted file mode 100644
index 8f85fe8..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCAggregator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.hllc;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-
-/**
- */
-@SuppressWarnings("serial")
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
-
- final int precision;
- HyperLogLogPlusCounter sum = null;
-
- public HLLCAggregator(int precision) {
- this.precision = precision;
- }
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(HyperLogLogPlusCounter value) {
- if (sum == null)
- sum = new HyperLogLogPlusCounter(value);
- else
- sum.merge(value);
- }
-
- @Override
- public HyperLogLogPlusCounter getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- // 1024 + 60 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 4 // precision
- + 8 // ref to HLLC
- + 8 // HLLC obj shell
- + 32 + (1 << precision); // HLLC internal
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
deleted file mode 100644
index 5612892..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/HLLCSerializer.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.hllc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
-
- private int precision;
-
- public HLLCSerializer(DataType type) {
- this.precision = type.getPrecision();
- }
-
- @Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
- try {
- value.writeRegisters(out);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private HyperLogLogPlusCounter current() {
- HyperLogLogPlusCounter hllc = current.get();
- if (hllc == null) {
- hllc = new HyperLogLogPlusCounter(precision);
- current.set(hllc);
- }
- return hllc;
- }
-
- @Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- HyperLogLogPlusCounter hllc = current();
- try {
- hllc.readRegisters(in);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return hllc;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return current().peekLength(in);
- }
-
- @Override
- public int maxLength() {
- return current().maxLength();
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return current().maxLength();
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(byte[] value) {
- HyperLogLogPlusCounter hllc = current();
- hllc.clear();
- if (value == null)
- hllc.add("__nUlL__");
- else
- hllc.add(value);
- return hllc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
deleted file mode 100644
index 151c1ee..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/hllc/LDCAggregator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.hllc;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.LongMutable;
-
-/**
- * Long Distinct Count
- */
-@SuppressWarnings("serial")
-public class LDCAggregator extends MeasureAggregator<LongMutable> {
-
- private static LongMutable ZERO = new LongMutable(0);
-
- private HLLCAggregator hllAgg = null;
- private LongMutable state = new LongMutable(0);
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- this.hllAgg = (HLLCAggregator) agg;
- }
-
- @Override
- public void reset() {
- }
-
- @Override
- public void aggregate(LongMutable value) {
- }
-
- @Override
- public LongMutable getState() {
- if (hllAgg == null) {
- return ZERO;
- } else {
- state.set(hllAgg.getState().getCountEstimate());
- return state;
- }
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
deleted file mode 100644
index 251abd9..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregation.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.topn;
-
-import java.util.List;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.aggregation.hllc.HLLCSerializer;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-public class TopNAggregation extends AggregationType {
-
- private final DataType dataType;
-
- public TopNAggregation(String dataType) {
- this.dataType = DataType.getType(dataType);
-
- if (this.dataType.getPrecision() < 1 || this.dataType.getPrecision() > 1000)
- throw new IllegalArgumentException("TopN precision must be between 1 and 1000");
- }
-
- @Override
- public DataType getAggregationDataType() {
- return dataType;
- }
-
- @Override
- public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
- return HLLCSerializer.class;
- }
-
- @Override
- public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public MeasureAggregator<?> newAggregator() {
- return new TopNAggregator();
- }
-
- @Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
- // TODO Auto-generated method stub
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
deleted file mode 100644
index 1ea22c8..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregationFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.topn;
-
-import org.apache.kylin.aggregation.AggregationType;
-import org.apache.kylin.aggregation.IAggregationFactory;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-public class TopNAggregationFactory implements IAggregationFactory {
-
- @Override
- public AggregationType createAggregationType(String funcName, String dataType) {
- if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName) == false)
- throw new IllegalArgumentException();
-
- return new TopNAggregation(dataType);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
deleted file mode 100644
index 4f6c7ee..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNAggregator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.topn;
-
-import java.util.Map;
-
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
-
- int capacity = 0;
- TopNCounter<ByteArray> sum = null;
- Map<ByteArray, Double> sanityCheckMap;
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(TopNCounter<ByteArray> value) {
- if (sum == null) {
- capacity = value.getCapacity();
- sum = new TopNCounter<>(capacity);
- sanityCheckMap = Maps.newHashMap();
- }
- sum.merge(value);
- }
-
- @Override
- public TopNCounter<ByteArray> getState() {
-
- //sum.retain(capacity);
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return 8 * capacity / 4;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java b/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
deleted file mode 100644
index 8088842..0000000
--- a/core-cube/src/main/java/org/apache/kylin/aggregation/topn/TopNCounterSerializer.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.topn;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.DoubleDeltaSerializer;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- *
- */
-public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteArray>> {
-
- private DoubleDeltaSerializer dds = new DoubleDeltaSerializer(3);
-
- private int precision;
-
- public TopNCounterSerializer(DataType dataType) {
- this.precision = dataType.getPrecision();
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- @SuppressWarnings("unused")
- int capacity = in.getInt();
- int size = in.getInt();
- int keyLength = in.getInt();
- dds.deserialize(in);
- int len = in.position() - mark + keyLength * size;
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8);
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return precision * TopNCounter.EXTRA_SPACE_RATE * 8;
- }
-
- @Override
- public TopNCounter<ByteArray> valueOf(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- int sizeOfId = buffer.getInt();
- int keyEncodedValue = buffer.getInt();
- double counter = buffer.getDouble();
-
- ByteArray key = new ByteArray(sizeOfId);
- BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, sizeOfId);
-
- TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(precision * TopNCounter.EXTRA_SPACE_RATE);
- topNCounter.offer(key, counter);
- return topNCounter;
- }
-
- @Override
- public void serialize(TopNCounter<ByteArray> value, ByteBuffer out) {
- double[] counters = value.getCounters();
- List<ByteArray> peek = value.peek(1);
- int keyLength = peek.size() > 0 ? peek.get(0).length() : 0;
- out.putInt(value.getCapacity());
- out.putInt(value.size());
- out.putInt(keyLength);
- dds.serialize(counters, out);
- Iterator<Counter<ByteArray>> iterator = value.iterator();
- while (iterator.hasNext()) {
- out.put(iterator.next().getItem().array());
- }
- }
-
- @Override
- public TopNCounter<ByteArray> deserialize(ByteBuffer in) {
- int capacity = in.getInt();
- int size = in.getInt();
- int keyLength = in.getInt();
- double[] counters = dds.deserialize(in);
-
- TopNCounter<ByteArray> counter = new TopNCounter<ByteArray>(capacity);
- ByteArray byteArray;
- for (int i = 0; i < size; i++) {
- byteArray = new ByteArray(keyLength);
- in.get(byteArray.array());
- counter.offerToHead(byteArray, counters[i]);
- }
-
- return counter;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4338f54..24286cc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -38,10 +38,10 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
@@ -182,7 +182,8 @@ public class CubeManager implements IRealizationProvider {
/**
* return null if no dictionary for given column
*/
- public Dictionary<?> getDictionary(CubeSegment cubeSeg, TblColRef col) {
+ @SuppressWarnings("unchecked")
+ public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) {
DictionaryInfo info = null;
try {
DictionaryManager dictMgr = getDictionaryManager();
@@ -199,7 +200,7 @@ public class CubeManager implements IRealizationProvider {
throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
}
- return info.getDictionaryObject();
+ return (Dictionary<String>) info.getDictionaryObject();
}
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 6d899f4..b29f83a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -25,11 +25,11 @@ import java.util.TimeZone;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -265,6 +265,14 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
public void setStorageLocationIdentifier(String storageLocationIdentifier) {
this.storageLocationIdentifier = storageLocationIdentifier;
}
+
+ public Map<TblColRef, Dictionary<String>> buildDictionaryMap() {
+ Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
+ for (TblColRef col : getCubeDesc().getAllColumnsNeedDictionary()) {
+ result.put(col, (Dictionary<String>) getDictionary(col));
+ }
+ return result;
+ }
@Override
public int getColumnLength(TblColRef col) {
@@ -277,7 +285,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
}
@Override
- public Dictionary<?> getDictionary(TblColRef col) {
+ public Dictionary<String> getDictionary(TblColRef col) {
return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
}
@@ -425,4 +433,5 @@ public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, I
public IJoinedFlatTableDesc getJoinedFlatTableDesc() {
return new CubeJoinedFlatTableDesc(this.getCubeDesc(), this);
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index 3619d69..7f38c26 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -1,23 +1,21 @@
package org.apache.kylin.cube.gridtable;
-import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
-import org.apache.kylin.common.datatype.StringSerializer;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.DefaultGTComparator;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.gridtable.IGTComparator;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
/**
* defines how column values will be encoded to/ decoded from GTRecord
@@ -111,9 +109,6 @@ public class CubeCodeSystem implements IGTCodeSystem {
if (serializer instanceof DictionarySerializer) {
((DictionarySerializer) serializer).serializeWithRounding(value, roundingFlag, buf);
} else {
- if ((value instanceof String) && (!(serializer instanceof StringSerializer || serializer instanceof FixLenSerializer))) {
- value = serializer.valueOf((String) value);
- }
serializer.serialize(value, buf);
}
}
@@ -176,11 +171,6 @@ public class CubeCodeSystem implements IGTCodeSystem {
}
@Override
- public Object valueOf(byte[] value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void serialize(Object value, ByteBuffer out) {
throw new UnsupportedOperationException();
}
@@ -230,10 +220,6 @@ public class CubeCodeSystem implements IGTCodeSystem {
return dictionary.getSizeOfId();
}
- @Override
- public Object valueOf(byte[] value) {
- throw new UnsupportedOperationException();
- }
}
static class FixLenSerializer extends DataTypeSerializer {
@@ -306,16 +292,6 @@ public class CubeCodeSystem implements IGTCodeSystem {
return fixLen;
}
- @Override
- public Object valueOf(byte[] value) {
- try {
- return new String(value, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- // does not happen
- throw new RuntimeException(e);
- }
- }
-
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index aa0a530..05fc8a5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -3,11 +3,11 @@ package org.apache.kylin.cube.gridtable;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.metadata.model.TblColRef;
@@ -16,24 +16,25 @@ import com.google.common.collect.Maps;
@SuppressWarnings("rawtypes")
public class CubeGridTable {
- public static Map<TblColRef, Dictionary<?>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) {
+ public static Map<TblColRef, Dictionary<String>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
// build a dictionary map
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+ Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
List<TblColRef> dimCols = Cuboid.findById(cubeDesc, cuboidId).getColumns();
for (TblColRef col : dimCols) {
- Dictionary<?> dictionary = cubeMgr.getDictionary(cubeSeg, col);
+ Dictionary<String> dictionary = cubeMgr.getDictionary(cubeSeg, col);
if (dictionary != null) {
dictionaryMap.put(col, dictionary);
}
}
+
return dictionaryMap;
}
public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException {
- Map<TblColRef, Dictionary<?>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
+ Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg, cuboidId);
Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
for (TblColRef dim : cuboid.getColumns()) {
if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
@@ -47,7 +48,7 @@ public class CubeGridTable {
return newGTInfo(cubeSeg.getCubeDesc(), cuboidId, dictionaryMap);
}
- public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary<String>> dictionaryMap) {
Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(cuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 8d132e2..361d96a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -6,11 +6,11 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
index 045b11e..9bbcf75 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
@@ -18,5 +18,6 @@
package org.apache.kylin.cube.gridtable;
+@SuppressWarnings("serial")
public class NotEnoughGTInfoException extends Exception {
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index d30186e..c4d0a7e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -23,12 +23,12 @@ package org.apache.kylin.cube.gridtable;
import java.nio.ByteBuffer;
import java.util.Map;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.gridtable.IGTComparator;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class TrimmedCubeCodeSystem implements IGTCodeSystem {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 0a35559..58f94c1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -21,8 +21,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GridTable;
@@ -39,12 +39,12 @@ abstract public class AbstractInMemCubeBuilder {
private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class);
final protected CubeDesc cubeDesc;
- final protected Map<TblColRef, Dictionary<?>> dictionaryMap;
+ final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected int taskThreadCount = 4;
protected int reserveMemoryMB = 200;
- public AbstractInMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ public AbstractInMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (cubeDesc == null)
throw new NullPointerException();
if (dictionaryMap == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index ce912a3..5b6131f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -29,15 +29,15 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.TimeUnit;
-import org.apache.kylin.aggregation.MeasureAggregators;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.measure.MeasureAggregators;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +54,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
private int splitRowThreshold = Integer.MAX_VALUE;
private int unitRows = 1000;
- public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
super(cubeDesc, dictionaryMap);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a393179..5c59de7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -28,9 +28,9 @@ import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.kylin.common.datatype.DoubleMutable;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -38,8 +38,6 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTAggregateScanner;
import org.apache.kylin.gridtable.GTBuilder;
import org.apache.kylin.gridtable.GTInfo;
@@ -47,6 +45,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -66,7 +65,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private final CuboidScheduler cuboidScheduler;
private final long baseCuboidId;
private final int totalCuboidCount;
- private final CubeJoinedFlatTableDesc intermediateTableDesc;
private final String[] metricsAggrFuncs;
private final MeasureDesc[] measureDescs;
private final int measureCount;
@@ -81,12 +79,11 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private Object[] totalSumForSanityCheck;
private ICuboidCollector resultCollector;
- public InMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ public InMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
super(cubeDesc, dictionaryMap);
this.cuboidScheduler = new CuboidScheduler(cubeDesc);
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
- this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
@@ -100,8 +97,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}
-
-
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
GTInfo info = CubeGridTable.newGTInfo(cubeDesc, cuboidID, dictionaryMap);
@@ -114,7 +109,6 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
return gridTable;
}
-
@Override
public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
ConcurrentNavigableMap<Long, CuboidResult> result = build(input);
@@ -447,7 +441,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
return updateCuboidResult(cuboidId, newGridTable, count, timeSpent, 0);
}
- //@SuppressWarnings("unused")
+ @SuppressWarnings({ "unused", "rawtypes", "unchecked" })
private void sanityCheck(long parentId, long cuboidId, Object[] totalSum) {
// double sum introduces error and causes result not exactly equal
for (int i = 0; i < totalSum.length; i++) {
@@ -508,9 +502,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.info = info;
this.input = input;
this.record = new GTRecord(info);
- this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc,
- InMemCubeBuilderUtils.createTopNLiteralColDictionaryMap(cubeDesc, intermediateTableDesc, dictionaryMap),
- info);
+ this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, dictionaryMap, info);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index bf4278a..fed9479 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -17,49 +17,39 @@
*/
package org.apache.kylin.cube.inmemcubing;
-import com.google.common.base.Preconditions;
+import java.util.List;
+import java.util.Map;
-import org.apache.kylin.aggregation.MeasureCodec;
-import org.apache.kylin.common.datatype.LongMutable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
/**
*/
public class InMemCubeBuilderInputConverter {
- private static final LongMutable ONE = new LongMutable(1l);
-
- private final CubeDesc cubeDesc;
private final CubeJoinedFlatTableDesc intermediateTableDesc;
private final MeasureDesc[] measureDescs;
- private final MeasureCodec measureCodec;
+ private final MeasureIngester<?>[] measureIngesters;
private final int measureCount;
- private final ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- private final Map<Integer, Dictionary<String>> topNLiteralColDictMap;
+ private final Map<TblColRef, Dictionary<String>> dictionaryMap;
private final GTInfo gtInfo;
- public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<Integer, Dictionary<String>> topNLiteralColDictMap, GTInfo gtInfo) {
- this.cubeDesc = cubeDesc;
+ public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) {
this.gtInfo = gtInfo;
this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
- this.measureCodec = new MeasureCodec(cubeDesc.getMeasures());
- this.topNLiteralColDictMap = Preconditions.checkNotNull(topNLiteralColDictMap, "topNLiteralColDictMap cannot be null");
+ this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+ this.dictionaryMap = dictionaryMap;
}
public final GTRecord convert(List<String> row) {
@@ -89,59 +79,38 @@ public class InMemCubeBuilderInputConverter {
}
private Object[] buildValue(List<String> row) {
-
Object[] values = new Object[measureCount];
for (int i = 0; i < measureCount; i++) {
- MeasureDesc measureDesc = measureDescs[i];
- int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[i];
- FunctionDesc function = cubeDesc.getMeasures().get(i).getFunction();
- if (flatTableIdx == null) {
- values[i] = measureCodec.getSerializer(i).valueOf(measureDesc.getFunction().getParameter().getValue());
- } else if (function.isCount() || function.isHolisticCountDistinct()) {
+ values[i] = buildValueOf(i, row);
+ }
+ return values;
+ }
+
+ private Object buildValueOf(int idxOfMeasure, List<String> row) {
+ MeasureDesc measure = measureDescs[idxOfMeasure];
+ FunctionDesc function = measure.getFunction();
+ int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
+
+ int paramCount = function.getParameterCount();
+ String[] inputToMeasure = new String[paramCount];
+
+ // pick up parameter values
+ ParameterDesc param = function.getParameter();
+ int paramColIdx = 0; // index among parameters of column type
+ for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
+ String value;
+ if (function.isCount() || function.isHolisticCountDistinct()) {
// note for holistic count distinct, this value will be ignored
- values[i] = ONE;
- } else if (function.isTopN()) {
- // encode the key column with dict, and get the counter column;
- int keyColIndex = flatTableIdx[flatTableIdx.length - 1];
- Dictionary<String> literalColDict = topNLiteralColDictMap.get(keyColIndex);
- int keyColEncoded = literalColDict.getIdFromValue(row.get(keyColIndex));
- valueBuf.clear();
- valueBuf.putInt(literalColDict.getSizeOfId());
- valueBuf.putInt(keyColEncoded);
- if (flatTableIdx.length == 1) {
- // only literalCol, use 1.0 as counter
- valueBuf.putDouble(1.0);
- } else {
- // get the counter column value
- valueBuf.putDouble(Double.valueOf(row.get(flatTableIdx[0])));
- }
-
- values[i] = measureCodec.getSerializer(i).valueOf(valueBuf.array());
-
- } else if (flatTableIdx.length == 1) {
- values[i] = measureCodec.getSerializer(i).valueOf(toBytes(row.get(flatTableIdx[0])));
+ value = "1";
+ } else if (param.isColumnType()) {
+ value = row.get(colIdxOnFlatTable[paramColIdx++]);
} else {
-
- byte[] result = null;
- for (int x = 0; x < flatTableIdx.length; x++) {
- byte[] split = toBytes(row.get(flatTableIdx[x]));
- if (result == null) {
- result = Arrays.copyOf(split, split.length);
- } else {
- byte[] newResult = new byte[result.length + split.length];
- System.arraycopy(result, 0, newResult, 0, result.length);
- System.arraycopy(split, 0, newResult, result.length, split.length);
- result = newResult;
- }
- }
- values[i] = measureCodec.getSerializer(i).valueOf(result);
+ value = param.getValue();
}
+ inputToMeasure[i] = value;
}
- return values;
- }
-
- private byte[] toBytes(String v) {
- return v == null ? null : Bytes.toBytes(v);
+
+ return measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
index d7d3f07..f8a887d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderUtils.java
@@ -18,11 +18,12 @@
package org.apache.kylin.cube.inmemcubing;
import com.google.common.collect.Maps;
+
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index 4316376..62432f7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -21,10 +21,10 @@ package org.apache.kylin.cube.kv;
import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index 7fedd90..ba15b48 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -22,7 +22,7 @@ import java.util.Arrays;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
index a21fe9f..fea3736 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
@@ -21,7 +21,7 @@ package org.apache.kylin.cube.kv;
import java.util.Collection;
import java.util.Comparator;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
/**
* @author yangli9
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 913ce3e..6124ac9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -42,6 +42,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -765,7 +766,8 @@ public class CubeDesc extends RootPersistentEntity {
}
for (MeasureDesc measure : measures) {
- result.addAll(measure.getColumnsNeedDictionary());
+ MeasureType aggrType = MeasureType.create(measure.getFunction());
+ result.addAll(aggrType.getColumnsNeedDictionary(measure));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index e4307e5..c918798 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -26,12 +26,12 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index ada31b6..f6b0e35 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -44,13 +44,13 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
@@ -148,7 +148,7 @@ public class CubingUtils {
return result;
}
- public static Map<TblColRef, Dictionary<?>> buildDictionary(final CubeInstance cubeInstance, Iterable<List<String>> recordList) throws IOException {
+ public static Map<TblColRef, Dictionary<String>> buildDictionary(final CubeInstance cubeInstance, Iterable<List<String>> recordList) throws IOException {
final List<TblColRef> columnsNeedToBuildDictionary = cubeInstance.getDescriptor().listDimensionColumnsExcludingDerived();
final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap();
int index = 0;
@@ -156,7 +156,7 @@ public class CubingUtils {
tblColRefMap.put(index++, column);
}
- HashMap<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+ HashMap<TblColRef, Dictionary<String>> result = Maps.newHashMap();
HashMultimap<TblColRef, String> valueMap = HashMultimap.create();
for (List<String> row : recordList) {
@@ -175,18 +175,19 @@ public class CubingUtils {
return input == null ? null : input.getBytes();
}
});
- final Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(tblColRef.getType(), new IterableDictionaryValueEnumerator(bytes));
+ final Dictionary<String> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(tblColRef.getType(), new IterableDictionaryValueEnumerator(bytes));
result.put(tblColRef, dict);
}
return result;
}
- public static Map<TblColRef, Dictionary<?>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<?>> dictionaryMap, long startOffset, long endOffset) {
- Map<TblColRef, Dictionary<?>> realDictMap = Maps.newHashMap();
+ @SuppressWarnings("unchecked")
+ public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) {
+ Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap();
- for (Map.Entry<TblColRef, Dictionary<?>> entry : dictionaryMap.entrySet()) {
+ for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) {
final TblColRef tblColRef = entry.getKey();
- final Dictionary<?> dictionary = entry.getValue();
+ final Dictionary<String> dictionary = entry.getValue();
ReadableTable.TableSignature signature = new ReadableTable.TableSignature();
signature.setLastModifiedTime(System.currentTimeMillis());
signature.setPath(String.format("streaming_%s_%s", startOffset, endOffset));
@@ -197,7 +198,7 @@ public class CubingUtils {
try {
DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo);
cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath());
- realDictMap.put(tblColRef, realDict.getDictionaryObject());
+ realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject());
} catch (IOException e) {
logger.error("error save dictionary for column:" + tblColRef, e);
throw new RuntimeException("error save dictionary for column:" + tblColRef, e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
index f7623e3..714571f 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/DefaultGTComparator.java
@@ -1,7 +1,7 @@
package org.apache.kylin.gridtable;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
public class DefaultGTComparator implements IGTComparator {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 01696e8..eb8d212 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -7,10 +7,10 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedMap;
-import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
+import org.apache.kylin.measure.MeasureAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 229c679..d3a03d1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -5,10 +5,10 @@ import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedList;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.cube.util.KryoUtils;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
public class GTInfo {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 2783f55..b3133be 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -2,9 +2,9 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DataTypeSerializer;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
@SuppressWarnings({ "rawtypes", "unchecked" })
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 0e61cf2..3a22091 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -2,8 +2,8 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
-import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.measure.MeasureAggregator;
public interface IGTCodeSystem {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index f8d7f30..ff71b4f 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -22,11 +22,11 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
public class UnitTestSupport {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
deleted file mode 100644
index 8ae44b6..0000000
--- a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package org.apache.kylin.aggregation.topn;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- *
- */
-public class TopNCounterSerializerTest {
-
- private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getType("topn(10)"));
-
- @Test
- public void testSerialization() {
- TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
- Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
- for (Integer i : stream) {
- vs.offer(new ByteArray(Bytes.toBytes(i)));
- }
-
- ByteBuffer out = ByteBuffer.allocate(1024);
- serializer.serialize(vs, out);
-
- byte[] copyBytes = new byte[out.position()];
- System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
-
- ByteBuffer in = ByteBuffer.wrap(copyBytes);
- TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
-
- Assert.assertEquals(vs.toString(), vsNew.toString());
-
- }
-
- @Test
- public void testValueOf() {
-
- TopNCounter<ByteArray> origin = new TopNCounter<ByteArray>(10);
- ByteArray key = new ByteArray(1);
- ByteBuffer byteBuffer = key.asBuffer();
- BytesUtil.writeVLong(20l, byteBuffer);
- origin.offer(key, 1.0);
-
- byteBuffer = ByteBuffer.allocate(1024);
- byteBuffer.putInt(1);
- byteBuffer.putInt(20);
- byteBuffer.putDouble(1.0);
- TopNCounter<ByteArray> counter = serializer.valueOf(byteBuffer.array());
-
-
- Assert.assertEquals(origin.toString(), counter.toString());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
index 0de5c22..e4dd8f5 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/DictionaryManagerTest.java
@@ -23,10 +23,10 @@ import static org.junit.Assert.assertTrue;
import java.util.HashSet;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
index 935e840..c25bad7 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java
@@ -26,10 +26,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
@@ -51,7 +51,7 @@ public class DoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
private static CubeInstance cube;
private static String flatTable;
- private static Map<TblColRef, Dictionary<?>> dictionaryMap;
+ private static Map<TblColRef, Dictionary<String>> dictionaryMap;
@BeforeClass
public static void before() throws IOException {
[10/21] kylin git commit: KYLIN-976 Add ingester;
Build part done, in-mem cube test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
index ed0a166..832584c 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderTest.java
@@ -33,10 +33,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
@@ -58,7 +58,7 @@ public class DoggedCubeBuilderTest extends LocalFileMetadataTestCase {
private static CubeInstance cube;
private static String flatTable;
- private static Map<TblColRef, Dictionary<?>> dictionaryMap;
+ private static Map<TblColRef, Dictionary<String>> dictionaryMap;
@BeforeClass
public static void before() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
index f853b08..ab87c2b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderTest.java
@@ -33,13 +33,13 @@ import java.util.concurrent.Future;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.gridtable.GTRecord;
@@ -66,7 +66,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
private static CubeInstance cube;
private static String flatTable;
- private static Map<TblColRef, Dictionary<?>> dictionaryMap;
+ private static Map<TblColRef, Dictionary<String>> dictionaryMap;
@BeforeClass
public static void before() throws IOException {
@@ -166,8 +166,8 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
queue.put(new ArrayList<String>(0));
}
- static Map<TblColRef, Dictionary<?>> getDictionaryMap(CubeInstance cube, String flatTable) throws IOException {
- Map<TblColRef, Dictionary<?>> result = Maps.newHashMap();
+ static Map<TblColRef, Dictionary<String>> getDictionaryMap(CubeInstance cube, String flatTable) throws IOException {
+ Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
CubeDesc desc = cube.getDescriptor();
CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(desc, null);
int nColumns = flatTableDesc.getColumnList().size();
@@ -178,7 +178,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
if (desc.getRowkey().isUseDictionary(col)) {
logger.info("Building dictionary for " + col);
List<byte[]> valueList = readValueList(flatTable, nColumns, flatTableDesc.getRowKeyColumnIndexes()[c]);
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(col.getType(), new IterableDictionaryValueEnumerator(valueList));
+ Dictionary<String> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(col.getType(), new IterableDictionaryValueEnumerator(valueList));
result.put(col, dict);
}
}
@@ -192,7 +192,7 @@ public class InMemCubeBuilderTest extends LocalFileMetadataTestCase {
TblColRef literalCol = func.getTopNLiteralColumn();
logger.info("Building dictionary for " + literalCol);
List<byte[]> valueList = readValueList(flatTable, nColumns, literalColIdx);
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(literalCol.getType(), new IterableDictionaryValueEnumerator(valueList));
+ Dictionary<String> dict = DictionaryGenerator.buildDictionaryFromValueEnumerator(literalCol.getType(), new IterableDictionaryValueEnumerator(valueList));
result.put(literalCol, dict);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 93f0419..6ae0d0e 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -23,15 +23,15 @@ import java.util.Random;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
-import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
-import org.apache.kylin.aggregation.basic.LongSumAggregator;
-import org.apache.kylin.aggregation.hllc.HLLCAggregator;
-import org.apache.kylin.common.datatype.DoubleMutable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.basic.BigDecimalSumAggregator;
+import org.apache.kylin.measure.basic.DoubleSumAggregator;
+import org.apache.kylin.measure.basic.LongSumAggregator;
+import org.apache.kylin.measure.hllc.HLLCAggregator;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.Test;
public class AggregationCacheMemSizeTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index b3981e8..836bafd 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -27,18 +27,18 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.gridtable.GTInfo.Builder;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 4313f4b..5454490 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -26,9 +26,9 @@ import java.math.BigDecimal;
import java.util.BitSet;
import java.util.List;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.Test;
public class SimpleGridTableTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 7f4da61..02fbecc 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -24,10 +24,10 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
-import org.apache.kylin.common.datatype.StringSerializer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.datatype.StringSerializer;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 4da5471..e9f74a4 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -23,11 +23,11 @@ import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
-import org.apache.kylin.aggregation.MeasureCodec;
-import org.apache.kylin.common.datatype.DoubleMutable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index b882e58..62b06aa 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -30,6 +30,7 @@ import java.io.UnsupportedEncodingException;
import java.util.Date;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.Dictionary;
/**
* A dictionary for date string (date only, no time).
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
index f19a2a8..b0326c1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
@@ -3,6 +3,7 @@ package org.apache.kylin.dict;
import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
deleted file mode 100644
index d1fc6f9..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.PrintStream;
-import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
-
-import org.apache.kylin.common.persistence.Writable;
-import org.apache.kylin.common.util.BytesUtil;
-
-/**
- * A bi-way dictionary that maps from dimension/column values to IDs and vice
- * versa. By storing IDs instead of real values, the size of cube is
- * significantly reduced.
- *
- * - IDs are smallest integers possible for the cardinality of a column, for the
- * purpose of minimal storage space - IDs preserve ordering of values, such that
- * range query can be applied to IDs directly
- *
- * A dictionary once built, is immutable. This allows optimal memory footprint
- * by e.g. flatten the Trie structure into a byte array, replacing node pointers
- * with array offsets.
- *
- * @author yangli9
- */
-@SuppressWarnings("serial")
-abstract public class Dictionary<T> implements Writable, Serializable {
-
- public static final byte NULL = (byte) 0xff;
-
- // ID with all bit-1 (0xff e.g.) reserved for NULL value
- public static final int NULL_ID[] = new int[] { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
-
- abstract public int getMinId();
-
- abstract public int getMaxId();
-
- public int getSize() {
- return getMaxId() - getMinId() + 1;
- }
-
- /**
- * @return the size of an ID in bytes, determined by the cardinality of column
- */
- abstract public int getSizeOfId();
-
- /**
- * @return the (maximum) size of value in bytes, determined by the longest value
- */
- abstract public int getSizeOfValue();
-
- /**
- * @return true if each entry of this dict is contained by the dict in param
- */
- abstract public boolean contains(Dictionary<?> another);
-
- /**
- * Convenient form of <code>getIdFromValue(value, 0)</code>
- */
- final public int getIdFromValue(T value) throws IllegalArgumentException {
- return getIdFromValue(value, 0);
- }
-
- /**
- * Returns the ID integer of given value. In case of not found
- * <p>
- * - if roundingFlag=0, throw IllegalArgumentException; <br>
- * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
- * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
- * <p>
- * The implementation often has cache, thus faster than the byte[] version getIdFromValueBytes()
- *
- * @throws IllegalArgumentException
- * if value is not found in dictionary and rounding is off;
- * or if rounding cannot find a smaller or bigger ID
- */
- final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
- if (isNullObjectForm(value))
- return nullId();
- else
- return getIdFromValueImpl(value, roundingFlag);
- }
-
- final public boolean containsValue(T value) throws IllegalArgumentException {
- if (isNullObjectForm(value)) {
- return true;
- } else {
- try {
- //if no key found, it will throw exception
- getIdFromValueImpl(value, 0);
- } catch (IllegalArgumentException e) {
- return false;
- }
- return true;
- }
- }
-
- protected boolean isNullObjectForm(T value) {
- return value == null;
- }
-
- abstract protected int getIdFromValueImpl(T value, int roundingFlag);
-
- /**
- * @return the value corresponds to the given ID
- * @throws IllegalArgumentException
- * if ID is not found in dictionary
- */
- final public T getValueFromId(int id) throws IllegalArgumentException {
- if (isNullId(id))
- return null;
- else
- return getValueFromIdImpl(id);
- }
-
- abstract protected T getValueFromIdImpl(int id);
-
- /**
- * Convenient form of
- * <code>getIdFromValueBytes(value, offset, len, 0)</code>
- */
- final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException {
- return getIdFromValueBytes(value, offset, len, 0);
- }
-
- /**
- * A lower level API, return ID integer from raw value bytes. In case of not found
- * <p>
- * - if roundingFlag=0, throw IllegalArgumentException; <br>
- * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
- * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
- * <p>
- * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
- *
- * @throws IllegalArgumentException
- * if value is not found in dictionary and rounding is off;
- * or if rounding cannot find a smaller or bigger ID
- */
- final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
- if (isNullByteForm(value, offset, len))
- return nullId();
- else {
- int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- if (id < 0)
- throw new IllegalArgumentException("Value not exists!");
- return id;
- }
- }
-
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null;
- }
-
- abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
-
- final public byte[] getValueBytesFromId(int id) {
- if (isNullId(id))
- return BytesUtil.EMPTY_BYTE_ARRAY;
- else
- return getValueBytesFromIdImpl(id);
- }
-
- abstract protected byte[] getValueBytesFromIdImpl(int id);
-
- /**
- * A lower level API, get byte values from ID, return the number of bytes
- * written. Bypassing the cache layer, this could be significantly slower
- * than getIdFromValue(T value).
- *
- * @return size of value bytes, 0 if empty string, -1 if null
- *
- * @throws IllegalArgumentException
- * if ID is not found in dictionary
- */
- final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
- if (isNullId(id))
- return -1;
- else
- return getValueBytesFromIdImpl(id, returnValue, offset);
- }
-
- abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
-
- abstract public void dump(PrintStream out);
-
- public int nullId() {
- return NULL_ID[getSizeOfId()];
- }
-
- public boolean isNullId(int id) {
- int nullId = NULL_ID[getSizeOfId()];
- return (nullId & id) == nullId;
- }
-
- /** utility that converts a dictionary ID to string, preserving order */
- public static String dictIdToString(byte[] idBytes, int offset, int length) {
- try {
- return new String(idBytes, offset, length, "ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- // never happen
- return null;
- }
- }
-
- /** the reverse of dictIdToString(), returns integer ID */
- public static int stringToDictId(String str) {
- try {
- byte[] bytes = str.getBytes("ISO-8859-1");
- return BytesUtil.readUnsigned(bytes, 0, bytes.length);
- } catch (UnsupportedEncodingException e) {
- // never happen
- return 0;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 300c240..4b01e60 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -26,9 +26,10 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.source.ReadableTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,7 +56,7 @@ public class DictionaryGenerator {
}
}
- public static Dictionary<?> buildDictionaryFromValueEnumerator(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ public static Dictionary<String> buildDictionaryFromValueEnumerator(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
Preconditions.checkNotNull(dataType, "dataType cannot be null");
Dictionary dict;
int baseId = 0; // always 0 for now
@@ -94,7 +95,7 @@ public class DictionaryGenerator {
return buildDictionaryFromValueEnumerator(dataType, new MultipleDictionaryValueEnumerator(sourceDicts));
}
- public static Dictionary<?> buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException {
+ public static Dictionary<String> buildDictionary(DictionaryInfo info, ReadableTable inpTable) throws IOException {
// currently all data types are casted to string to build dictionary
// String dataType = info.getDataType();
@@ -111,7 +112,7 @@ public class DictionaryGenerator {
}
}
- private static Dictionary buildDateDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException {
+ private static Dictionary<String> buildDateDict(IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList samples) throws IOException {
final int BAD_THRESHOLD = 0;
String matchPattern = null;
byte[] value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index f482002..4fba59a 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -20,6 +20,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.source.ReadableTable.TableSignature;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
index 5b7a318..69b29fe 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.JsonUtil;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index f3cab64..f2bfd91 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -30,9 +30,10 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
index b80f838..6b47868 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionarySerializer.java
@@ -10,6 +10,7 @@ import java.io.OutputStream;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
index 1c7a009..4586163 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryAware.java
@@ -18,6 +18,7 @@
package org.apache.kylin.dict;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.TblColRef;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
index 13f7394..df7b1c6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/MultipleDictionaryValueEnumerator.java
@@ -19,7 +19,9 @@
package org.apache.kylin.dict;
import com.google.common.collect.Lists;
+
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
import java.io.IOException;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index 6bae129..65c6c05 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -7,6 +7,7 @@ import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Dictionary;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 3a05d0a..552aa92 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -19,9 +19,11 @@
package org.apache.kylin.dict;
import com.google.common.base.Preconditions;
+
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index e9a99b9..6297906 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -30,7 +30,7 @@ import java.util.List;
import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.dict.TrieDictionaryBuilder;
@@ -44,6 +44,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* @author yangli9
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class SnapshotTable extends RootPersistentEntity implements ReadableTable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 8c8dcfc..8020729 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -28,8 +28,9 @@ import java.util.List;
import java.util.Random;
import java.util.Set;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.datatype.DataType;
import org.junit.Test;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
new file mode 100644
index 0000000..0ab547a
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/IMeasureFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+public interface IMeasureFactory {
+
+ public MeasureType createMeasureType(String funcName, String dataType);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
new file mode 100644
index 0000000..32e5128
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.io.Serializable;
+
+/**
+ */
+@SuppressWarnings("serial")
+abstract public class MeasureAggregator<V> implements Serializable {
+
+ public static MeasureAggregator<?> create(String funcName, String dataType) {
+ return MeasureType.create(funcName, dataType).newAggregator();
+ }
+
+ public static int guessBigDecimalMemBytes() {
+ // 116 returned by AggregationCacheMemSizeTest
+ return 8 // aggregator obj shell
+ + 8 // ref to BigDecimal
+ + 8 // BigDecimal obj shell
+ + 100; // guess of BigDecimal internal
+ }
+
+ public static int guessDoubleMemBytes() {
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to DoubleWritable
+ + 8 // DoubleWritable obj shell
+ + 8; // size of double
+ */
+ }
+
+ public static int guessLongMemBytes() {
+ // 29 to 44 returned by AggregationCacheMemSizeTest
+ return 44;
+ /*
+ return 8 // aggregator obj shell
+ + 8 // ref to LongWritable
+ + 8 // LongWritable obj shell
+ + 8; // size of long
+ */
+ }
+
+ // ============================================================================
+
+ @SuppressWarnings("rawtypes")
+ public void setDependentAggregator(MeasureAggregator agg) {
+ }
+
+ abstract public void reset();
+
+ abstract public void aggregate(V value);
+
+ abstract public V getState();
+
+ // get an estimate of memory consumption UPPER BOUND
+ abstract public int getMemBytesEstimate();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
new file mode 100644
index 0000000..c6b456e
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+/**
+ */
+@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
+public class MeasureAggregators implements Serializable {
+
+ private final MeasureAggregator[] aggs;
+ private final int descLength;
+
+ public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
+ this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
+ }
+
+ public MeasureAggregators(MeasureDesc... measureDescs) {
+ descLength = measureDescs.length;
+ aggs = new MeasureAggregator[descLength];
+
+ Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
+ for (int i = 0; i < descLength; i++) {
+ FunctionDesc func = measureDescs[i].getFunction();
+ aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
+ measureIndexMap.put(measureDescs[i].getName(), i);
+ }
+ // fill back dependent aggregator
+ for (int i = 0; i < descLength; i++) {
+ String depMsrRef = measureDescs[i].getDependentMeasureRef();
+ if (depMsrRef != null) {
+ int index = measureIndexMap.get(depMsrRef);
+ aggs[i].setDependentAggregator(aggs[index]);
+ }
+ }
+ }
+
+ public void reset() {
+ for (int i = 0; i < aggs.length; i++) {
+ aggs[i].reset();
+ }
+ }
+
+ public void aggregate(Object[] values) {
+ assert values.length == descLength;
+
+ for (int i = 0; i < descLength; i++) {
+ aggs[i].aggregate(values[i]);
+ }
+ }
+
+ public void collectStates(Object[] states) {
+ for (int i = 0; i < descLength; i++) {
+ states[i] = aggs[i].getState();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
new file mode 100644
index 0000000..b02addd
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+public class MeasureCodec {
+
+ int nMeasures;
+ DataTypeSerializer[] serializers;
+
+ public MeasureCodec(Collection<MeasureDesc> measureDescs) {
+ this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
+ }
+
+ public MeasureCodec(MeasureDesc... measureDescs) {
+ String[] dataTypes = new String[measureDescs.length];
+ for (int i = 0; i < dataTypes.length; i++) {
+ dataTypes[i] = measureDescs[i].getFunction().getReturnType();
+ }
+ init(dataTypes);
+ }
+
+ public MeasureCodec(String... dataTypes) {
+ init(dataTypes);
+ }
+
+ private void init(String[] dataTypes) {
+ nMeasures = dataTypes.length;
+ serializers = new DataTypeSerializer[nMeasures];
+
+ for (int i = 0; i < nMeasures; i++) {
+ serializers[i] = DataTypeSerializer.create(dataTypes[i]);
+ }
+ }
+
+ public DataTypeSerializer getSerializer(int idx) {
+ return serializers[idx];
+ }
+
+ public void decode(ByteBuffer buf, Object[] result) {
+ assert result.length == nMeasures;
+ for (int i = 0; i < nMeasures; i++) {
+ result[i] = serializers[i].deserialize(buf);
+ }
+ }
+
+ public void encode(Object[] values, ByteBuffer out) {
+ assert values.length == nMeasures;
+ for (int i = 0; i < nMeasures; i++) {
+ serializers[i].serialize(values[i], out);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
new file mode 100644
index 0000000..8d6e601
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+abstract public class MeasureIngester<V> {
+
+ public static MeasureIngester<?> create(MeasureDesc measure) {
+ return MeasureType.create(measure.getFunction()).newIngester();
+ }
+
+ public static MeasureIngester<?>[] create(Collection<MeasureDesc> measures) {
+ MeasureIngester<?>[] result = new MeasureIngester<?>[measures.size()];
+ int i = 0;
+ for (MeasureDesc measure : measures) {
+ result[i++] = create(measure);
+ }
+ return result;
+ }
+
+ abstract public V valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
new file mode 100644
index 0000000..604a7b6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.basic.BasicMeasureFactory;
+import org.apache.kylin.measure.hllc.HLLCAggregationFactory;
+import org.apache.kylin.measure.topn.TopNMeasureFactory;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Maps;
+
+abstract public class MeasureType {
+
+ private static final Map<String, IMeasureFactory> factoryRegistry = Maps.newConcurrentMap();
+ private static final IMeasureFactory defaultFactory = new BasicMeasureFactory();
+
+ static {
+ factoryRegistry.put(FunctionDesc.FUNC_COUNT_DISTINCT, new HLLCAggregationFactory());
+ factoryRegistry.put(FunctionDesc.FUNC_TOP_N, new TopNMeasureFactory());
+ }
+
+ public static MeasureType create(FunctionDesc function) {
+ return create(function.getExpression(), function.getReturnType());
+ }
+
+ public static MeasureType create(String funcName, String dataType) {
+ funcName = funcName.toUpperCase();
+ dataType = dataType.toLowerCase();
+
+ IMeasureFactory factory = factoryRegistry.get(funcName);
+ if (factory == null)
+ factory = defaultFactory;
+
+ MeasureType result = factory.createMeasureType(funcName, dataType);
+
+ // register serializer for aggr data type
+ DataType aggregationDataType = result.getAggregationDataType();
+ if (DataTypeSerializer.hasRegistered(aggregationDataType.getName()) == false) {
+ DataTypeSerializer.register(aggregationDataType.getName(), result.getAggregationDataSeralizer());
+ }
+
+ return result;
+ }
+
+ /* ============================================================================
+ * Define
+ * ---------------------------------------------------------------------------- */
+
+ abstract public DataType getAggregationDataType();
+
+ abstract public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer();
+
+ abstract public void validate(MeasureDesc measureDesc) throws IllegalArgumentException;
+
+ /* ============================================================================
+ * Build
+ * ---------------------------------------------------------------------------- */
+
+ abstract public MeasureIngester<?> newIngester();
+
+ abstract public MeasureAggregator<?> newAggregator();
+
+ abstract public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc);
+
+ abstract public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts);
+
+ /* ============================================================================
+ * Cube Selection
+ * ---------------------------------------------------------------------------- */
+
+ /* ============================================================================
+ * Query
+ * ---------------------------------------------------------------------------- */
+
+ /* ============================================================================
+ * Storage
+ * ---------------------------------------------------------------------------- */
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
new file mode 100644
index 0000000..7bfee49
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.IMeasureFactory;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
+public class BasicMeasureFactory implements IMeasureFactory {
+
+ @Override
+ public MeasureType createMeasureType(String funcName, String dataType) {
+ return new BasicMeasureType(funcName, DataType.getType(dataType));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
new file mode 100644
index 0000000..f6bf090
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.util.List;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.datatype.BigDecimalSerializer;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.datatype.DoubleSerializer;
+import org.apache.kylin.metadata.datatype.LongSerializer;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class BasicMeasureType extends MeasureType {
+
+ private final String funcName;
+ private final DataType dataType;
+
+ public BasicMeasureType(String funcName, DataType dataType) {
+ this.funcName = funcName;
+ this.dataType = dataType;
+ }
+
+ @Override
+ public DataType getAggregationDataType() {
+ return dataType;
+ }
+
+ public Class<? extends DataTypeSerializer<?>> getAggregationDataSeralizer() {
+ if (dataType.isIntegerFamily())
+ return LongSerializer.class;
+ else if (dataType.isDecimal())
+ return BigDecimalSerializer.class;
+ else if (dataType.isNumberFamily())
+ return DoubleSerializer.class;
+ else
+ throw new IllegalArgumentException("No serializer for aggregation type " + dataType);
+ }
+
+ @Override
+ public void validate(MeasureDesc measureDesc) throws IllegalArgumentException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public MeasureIngester<?> newIngester() {
+ if (dataType.isIntegerFamily())
+ return new LongIngester();
+ else if (dataType.isDecimal())
+ return new BigDecimalIngester();
+ else if (dataType.isNumberFamily())
+ return new DoubleIngester();
+ else
+ throw new IllegalArgumentException("No ingester for aggregation type " + dataType);
+ }
+
+ @Override
+ public MeasureAggregator<?> newAggregator() {
+ if (isSum() || isCount()) {
+ if (dataType.isDecimal())
+ return new BigDecimalSumAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongSumAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleSumAggregator();
+ } else if (isMax()) {
+ if (dataType.isDecimal())
+ return new BigDecimalMaxAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongMaxAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleMaxAggregator();
+ } else if (isMin()) {
+ if (dataType.isDecimal())
+ return new BigDecimalMinAggregator();
+ else if (dataType.isIntegerFamily())
+ return new LongMinAggregator();
+ else if (dataType.isNumberFamily())
+ return new DoubleMinAggregator();
+ }
+ throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + dataType + "'");
+ }
+
+ private boolean isSum() {
+ return FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isCount() {
+ return FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isMax() {
+ return FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName);
+ }
+
+ private boolean isMin() {
+ return FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName);
+ }
+
+ @Override
+ public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Object reEncodeDictionary(Object value, List<Dictionary<?>> oldDicts, List<Dictionary<?>> newDicts) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
new file mode 100644
index 0000000..bb743d6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.math.BigDecimal;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class BigDecimalIngester extends MeasureIngester<BigDecimal> {
+
+ public BigDecimal valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (values.length > 1)
+ throw new IllegalArgumentException();
+
+ if (values[0] == null)
+ return new BigDecimal(0);
+ else
+ return new BigDecimal(values[0]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
new file mode 100644
index 0000000..aa42476
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.measure.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ if (max == null)
+ max = value;
+ else if (max.compareTo(value) < 0)
+ max = value;
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
new file mode 100644
index 0000000..81193ad
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.measure.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ if (max == null)
+ max = value;
+ else if (max.compareTo(value) > 0)
+ max = value;
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
new file mode 100644
index 0000000..5e00c63
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.math.BigDecimal;
+
+import org.apache.kylin.measure.MeasureAggregator;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
+
+ BigDecimal sum = new BigDecimal(0);
+
+ @Override
+ public void reset() {
+ sum = new BigDecimal(0);
+ }
+
+ @Override
+ public void aggregate(BigDecimal value) {
+ sum = sum.add(value);
+ }
+
+ @Override
+ public BigDecimal getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessBigDecimalMemBytes();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
new file mode 100644
index 0000000..506ed19
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class DoubleIngester extends MeasureIngester<DoubleMutable> {
+
+ // avoid repeated object creation
+ private DoubleMutable current = new DoubleMutable();
+
+ public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (values.length > 1)
+ throw new IllegalArgumentException();
+
+ DoubleMutable l = current;
+ if (values[0] == null)
+ l.set(0L);
+ else
+ l.set(Double.parseDouble(values[0]));
+ return l;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
new file mode 100644
index 0000000..25911e8
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ if (max == null)
+ max = new DoubleMutable(value.get());
+ else if (max.get() < value.get())
+ max.set(value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
new file mode 100644
index 0000000..be97deb
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable min = null;
+
+ @Override
+ public void reset() {
+ min = null;
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ if (min == null)
+ min = new DoubleMutable(value.get());
+ else if (min.get() > value.get())
+ min.set(value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return min;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
new file mode 100644
index 0000000..f276817
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
+
+ DoubleMutable sum = new DoubleMutable();
+
+ @Override
+ public void reset() {
+ sum.set(0.0);
+ }
+
+ @Override
+ public void aggregate(DoubleMutable value) {
+ sum.set(sum.get() + value.get());
+ }
+
+ @Override
+ public DoubleMutable getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessDoubleMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
new file mode 100644
index 0000000..5bf1257
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class LongIngester extends MeasureIngester<LongMutable> {
+
+ // avoid repeated object creation
+ private LongMutable current = new LongMutable();
+
+ public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (values.length > 1)
+ throw new IllegalArgumentException();
+
+ LongMutable l = current;
+ if (values[0] == null)
+ l.set(0L);
+ else
+ l.set(Long.parseLong(values[0]));
+ return l;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
new file mode 100644
index 0000000..ca44f15
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable max = null;
+
+ @Override
+ public void reset() {
+ max = null;
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ if (max == null)
+ max = new LongMutable(value.get());
+ else if (max.get() < value.get())
+ max.set(value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return max;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
new file mode 100644
index 0000000..dadc64e
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongMinAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable min = null;
+
+ @Override
+ public void reset() {
+ min = null;
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ if (min == null)
+ min = new LongMutable(value.get());
+ else if (min.get() > value.get())
+ min.set(value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return min;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
new file mode 100644
index 0000000..e7fdc9d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.basic;
+
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.LongMutable;
+
+/**
+ */
+@SuppressWarnings("serial")
+public class LongSumAggregator extends MeasureAggregator<LongMutable> {
+
+ LongMutable sum = new LongMutable();
+
+ @Override
+ public void reset() {
+ sum.set(0);
+ }
+
+ @Override
+ public void aggregate(LongMutable value) {
+ sum.set(sum.get() + value.get());
+ }
+
+ @Override
+ public LongMutable getState() {
+ return sum;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return guessLongMemBytes();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
new file mode 100644
index 0000000..13e5520
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregationFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import org.apache.kylin.measure.IMeasureFactory;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
+
+public class HLLCAggregationFactory implements IMeasureFactory {
+
+ @Override
+ public MeasureType createMeasureType(String funcName, String dataType) {
+ if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName) == false)
+ throw new IllegalArgumentException();
+
+ return new HLLCMeasureType(DataType.getType(dataType));
+ }
+
+}
[05/21] kylin git commit: KYLIN-976 AggregationType interface,
serializer and aggregator
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index e8b21d3..2717040 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
@@ -60,7 +61,6 @@ import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.CuboidShardUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index 54217ff..b247ad2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -24,9 +24,9 @@ import java.util.Collection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.aggregation.MeasureCodec;
+import org.apache.kylin.common.datatype.DoubleMutable;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DoubleMutable;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 08d4a49..873fb37 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,8 +35,8 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
index 8e665f7..e2e5724 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
@@ -21,8 +21,8 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import com.google.common.collect.Lists;
import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2d82d38/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index d534e44..847ad88 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -26,8 +26,8 @@ import java.util.Arrays;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.aggregation.MeasureCodec;
+import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
[08/21] kylin git commit: KYLIN-976 Add ingester;
Build part done, in-mem cube test pass
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 55235be..e79bd3a 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -31,13 +31,13 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.aggregation.MeasureCodec;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 5145efc..5db3631 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -29,19 +29,19 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.source.ReadableTable.TableSignature;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 4159718..8f65691 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -39,12 +39,11 @@ import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.aggregation.MeasureAggregators;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -57,12 +56,13 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.*;
import org.apache.kylin.cube.util.CubingUtils;
import org.apache.kylin.dict.*;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.engine.spark.cube.BufferedCuboidWriter;
import org.apache.kylin.engine.spark.cube.DefaultTupleConverter;
import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -162,7 +162,7 @@ public class SparkCubing extends AbstractApplication {
tblColRefMap.put(rowKeyColumnIndex, col);
}
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+ Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
for (Map.Entry<Integer, TblColRef> entry : tblColRefMap.entrySet()) {
final String column = columns[entry.getKey()];
final TblColRef tblColRef = entry.getValue();
@@ -308,12 +308,12 @@ public class SparkCubing extends AbstractApplication {
for (TblColRef tblColRef : baseCuboidColumn) {
columnLengthMap.put(tblColRef, cubeSegment.getColumnLength(tblColRef));
}
- final Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+ final Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
for (DimensionDesc dim : cubeDesc.getDimensions()) {
// dictionary
for (TblColRef col : dim.getColumnRefs()) {
if (cubeDesc.getRowkey().isUseDictionary(col)) {
- Dictionary<?> dict = cubeSegment.getDictionary(col);
+ Dictionary<String> dict = cubeSegment.getDictionary(col);
if (dict == null) {
System.err.println("Dictionary for " + col + " was not found.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
index 4a3b8b8..1965f8c 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/OneOffStreamingBuilder.java
@@ -36,8 +36,8 @@ package org.apache.kylin.engine.streaming;
import java.util.Map;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.StreamingBatch;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.engine.streaming.util.StreamingUtils;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.TblColRef;
@@ -72,7 +72,7 @@ public class OneOffStreamingBuilder {
StreamingBatch streamingBatch = streamingInput.getBatchWithTimeWindow(streamingConfig, -1, startTime, endTime);
final IBuildable buildable = streamingBatchBuilder.createBuildable(streamingBatch);
final Map<Long, HyperLogLogPlusCounter> samplingResult = streamingBatchBuilder.sampling(streamingBatch);
- final Map<TblColRef, Dictionary<?>> dictionaryMap = streamingBatchBuilder.buildDictionary(streamingBatch, buildable);
+ final Map<TblColRef, Dictionary<String>> dictionaryMap = streamingBatchBuilder.buildDictionary(streamingBatch, buildable);
streamingBatchBuilder.build(streamingBatch, dictionaryMap, streamingOutput.getCuboidWriter(buildable));
streamingOutput.output(buildable, samplingResult);
streamingBatchBuilder.commit(buildable);
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
index 93cda2d..91cdea9 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingBatchBuilder.java
@@ -36,9 +36,9 @@ package org.apache.kylin.engine.streaming;
import java.util.Map;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.StreamingBatch;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.TblColRef;
@@ -50,9 +50,9 @@ public interface StreamingBatchBuilder {
Map<Long, HyperLogLogPlusCounter> sampling(StreamingBatch streamingBatch);
- Map<TblColRef, Dictionary<?>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable);
+ Map<TblColRef, Dictionary<String>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable);
- void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<?>> dictionaryMap, ICuboidWriter cuboidWriter);
+ void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter);
void commit(IBuildable buildable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index ae72218..044dcca 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -53,7 +53,7 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder;
import org.apache.kylin.cube.util.CubingUtils;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.StreamingBatch;
import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
import org.apache.kylin.common.util.StreamingMessage;
@@ -79,7 +79,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
}
@Override
- public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<?>> dictionaryMap, ICuboidWriter cuboidWriter) {
+ public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>> dictionaryMap, ICuboidWriter cuboidWriter) {
try {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
@@ -130,10 +130,10 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
}
@Override
- public Map<TblColRef, Dictionary<?>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable) {
+ public Map<TblColRef, Dictionary<String>> buildDictionary(StreamingBatch streamingBatch, IBuildable buildable) {
final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
- final Map<TblColRef, Dictionary<?>> dictionaryMap;
+ final Map<TblColRef, Dictionary<String>> dictionaryMap;
try {
dictionaryMap = CubingUtils.buildDictionary(cubeInstance,
Lists.transform(streamingBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
@@ -143,7 +143,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
return input.getData();
}
}));
- Map<TblColRef, Dictionary<?>> realDictMap = CubingUtils.writeDictionary((CubeSegment) buildable,
+ Map<TblColRef, Dictionary<String>> realDictMap = CubingUtils.writeDictionary((CubeSegment) buildable,
dictionaryMap,
streamingBatch.getTimeRange().getFirst(),
streamingBatch.getTimeRange().getSecond());
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
index c3ca464..dedf8de 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
@@ -22,7 +22,7 @@ import java.text.SimpleDateFormat;
import java.util.List;
import java.util.TimeZone;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.model.IIDesc;
@@ -36,6 +36,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Objects;
+
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.IRealizationSegment;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
index c32b767..e395544 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
@@ -24,10 +24,11 @@ import java.util.Arrays;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import com.ning.compress.lzf.LZFDecoder;
import com.ning.compress.lzf.LZFEncoder;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index aad1fa2..2d635ab 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -21,11 +21,11 @@ package org.apache.kylin.invertedindex.index;
import java.util.Arrays;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index 16fb3a8..dc2c5c4 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -21,9 +21,10 @@ package org.apache.kylin.invertedindex.index;
import java.util.Iterator;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import com.google.common.base.Objects;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
index 4b9d7f5..792a0cc 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
@@ -19,7 +19,8 @@ package org.apache.kylin.invertedindex.index;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import org.apache.kylin.dict.Dictionary;
+
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.StreamingBatch;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.invertedindex.model.IIDesc;
@@ -28,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 12fcf29..3ee34be 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -22,10 +22,10 @@ import java.util.Arrays;
import org.apache.commons.lang.ObjectUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.metadata.datatype.LongMutable;
/**
* TableRecord extends RawTableRecord by decorating it with a TableRecordInfo.
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index d9e5b26..628a08d 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -20,12 +20,12 @@ package org.apache.kylin.invertedindex.index;
import java.util.List;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Array;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
index afb3633..09e6e4b 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
@@ -21,11 +21,11 @@ package org.apache.kylin.invertedindex.index;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
import com.google.common.base.Objects;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
index 42e5185..8671f43 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java
@@ -19,8 +19,8 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.datatype.DataType;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
index 4fa8714..c9a78a4 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java
@@ -20,7 +20,7 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
abstract public class FixedLenMeasureCodec<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
index 2b65192..67e5158 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java
@@ -20,9 +20,9 @@ package org.apache.kylin.invertedindex.measure;
import java.nio.ByteBuffer;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 0f29d95..73a605c 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -23,17 +23,17 @@ import java.util.Collection;
import java.util.Iterator;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.DictionarySerializer;
import org.apache.kylin.invertedindex.index.ColumnValueContainer;
import org.apache.kylin.invertedindex.index.CompressedValueContainer;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
+import org.apache.kylin.metadata.datatype.DataType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
index 2474750..4187193 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java
@@ -40,7 +40,7 @@ import java.util.List;
import javax.annotation.Nullable;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.invertedindex.model.IIDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
index f30a876..6047514 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java
@@ -21,8 +21,8 @@ package org.apache.kylin.invertedindex;
import java.io.IOException;
import java.util.List;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
index d72899b..8bd19ab 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java
@@ -34,8 +34,8 @@ import javax.annotation.Nullable;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.invertedindex.IIInstance;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
index 600d034..03b4068 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java
@@ -1,6 +1,6 @@
package org.apache.kylin.invertedindex.measure;
-import org.apache.kylin.common.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataType;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
index 1a6d944..06cdbba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
@@ -20,8 +20,8 @@ package org.apache.kylin.storage.hbase.common.coprocessor;
import java.util.Map;
-import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.MemoryBudgetController;
+import org.apache.kylin.measure.MeasureAggregator;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 9b12bfd..c833781 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -4,9 +4,9 @@ import java.util.Collection;
import java.util.Set;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.dict.DictCodeSystem;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 652d558..84db027 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -36,6 +36,7 @@ import java.util.TreeSet;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -46,7 +47,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index 6a15bc3..a5f7c74 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -13,12 +13,12 @@ import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowKeyDecoder;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index 7a75bbc..d8b61b3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index b75510f..6b27aaa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
index c9343fb..521f111 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
@@ -25,13 +25,13 @@ import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
index 4952931..06c6e2c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
@@ -19,7 +19,7 @@
package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index f09ce8a..be523d7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -29,12 +29,12 @@ import org.apache.kylin.common.topn.TopNCounter;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java
index 87d1b49..1f024fe 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluator.java
@@ -20,7 +20,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.util.List;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java
index a14f3da..41a2584 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/ClearTextDictionary.java
@@ -36,7 +36,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.util.Map;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
index a9d3b10..5b82aa4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationCache.java
@@ -21,7 +21,7 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import java.util.Map;
import java.util.Set;
-import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
index 910ed31..036d9b8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregators.java
@@ -22,9 +22,6 @@ import java.nio.ByteBuffer;
import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.DataType;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
@@ -32,6 +29,9 @@ import org.apache.kylin.invertedindex.index.RawTableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
index 046d8c1..0fe9898 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/IIEndpoint.java
@@ -36,18 +36,18 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.invertedindex.index.RawTableRecord;
import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorConstants;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
index b44ae7e..94e5b33 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/LocalDictionary.java
@@ -1,6 +1,6 @@
package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.IDictionaryAware;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 2717040..a7d0776 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.datatype.DataType;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ByteArray;
@@ -60,6 +59,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.CuboidShardUtil;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
index c834ee2..8205ff7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -34,6 +33,7 @@ import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.measure.MeasureCodec;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index 691258d..f190186 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
@@ -59,6 +58,7 @@ import org.apache.kylin.engine.mr.IMROutput2;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.steps.MergeCuboidMapper;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
index 4037f90..dc35c56 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/KeyValueCreator.java
@@ -5,11 +5,11 @@ import java.util.List;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.io.Text;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.model.MeasureDesc;
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index b247ad2..617af76 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -23,11 +23,11 @@ import java.util.BitSet;
import java.util.Collection;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.kylin.aggregation.MeasureCodec;
-import org.apache.kylin.common.datatype.DoubleMutable;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.measure.MeasureCodec;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 873fb37..cd4e33d 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,9 +35,9 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java
index da1cf71..61df5cf 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/BitMapFilterEvaluatorTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
-import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -36,6 +36,7 @@ import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.BitMapFilterEvalua
import org.junit.Test;
import com.google.common.collect.Lists;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
public class BitMapFilterEvaluatorTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
index e2e5724..04e1f7f 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointAggregationTest.java
@@ -20,14 +20,14 @@ package org.apache.kylin.storage.hbase.ii.coprocessor.endpoint;
import com.google.common.collect.Lists;
-import org.apache.kylin.aggregation.MeasureAggregator;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
index cf706e4..b715498 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java
@@ -29,13 +29,13 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.measure.MeasureCodec;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/99396828/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index 847ad88..488ff59 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -25,14 +25,14 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.aggregation.MeasureCodec;
-import org.apache.kylin.common.datatype.LongMutable;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.measure.MeasureCodec;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
[17/21] kylin git commit: KYLIN-976 Extract fillTuple() methods
Posted by li...@apache.org.
KYLIN-976 Extract fillTuple() methods
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/636aeecf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/636aeecf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/636aeecf
Branch: refs/heads/2.x-staging
Commit: 636aeecf2d55cc016b188fceac9fe44e58aa3ee3
Parents: 98a3e4c
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Dec 3 14:10:33 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 5 06:52:11 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/topn/TopNCounter.java | 4 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
core-metadata/pom.xml | 6 +
.../org/apache/kylin/measure/MeasureType.java | 33 ++-
.../kylin/measure/basic/BasicMeasureType.java | 2 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 2 +-
.../kylin/measure/topn/TopNMeasureType.java | 95 ++++++++-
.../kylin/metadata/model/FunctionDesc.java | 2 +-
.../org/apache/kylin/metadata/tuple/Tuple.java | 209 ++++++++++++++++++
.../apache/kylin/metadata/tuple/TupleInfo.java | 114 ++++++++++
core-storage/pom.xml | 7 -
.../org/apache/kylin/storage/IStorageQuery.java | 2 +-
.../storage/cache/CacheFledgedDynamicQuery.java | 2 +-
.../storage/cache/CacheFledgedStaticQuery.java | 2 +-
.../kylin/storage/cache/StreamSQLResult.java | 2 +-
.../storage/hybrid/HybridStorageQuery.java | 2 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 210 -------------------
.../apache/kylin/storage/tuple/TupleInfo.java | 114 ----------
.../kylin/storage/cache/DynamicCacheTest.java | 4 +-
.../kylin/storage/cache/StaticCacheTest.java | 4 +-
.../kylin/storage/cache/StorageMockUtils.java | 2 +-
.../mr/steps/MergeCuboidFromStorageMapper.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 2 +-
pom.xml | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 5 +-
.../apache/kylin/query/relnode/OLAPContext.java | 3 +-
server/pom.xml | 2 +-
.../org/apache/kylin/rest/StorageCacheTest.java | 2 +-
.../cube/v1/CubeSegmentTopNTupleIterator.java | 4 +-
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 4 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 4 +-
.../cube/v1/SerializedHBaseTupleIterator.java | 2 +-
.../storage/hbase/cube/v2/CubeStorageQuery.java | 34 +--
.../hbase/cube/v2/CubeTupleConverter.java | 82 ++++++--
.../v2/SequentialCubeTopNTupleIterator.java | 4 +-
.../cube/v2/SequentialCubeTupleIterator.java | 56 ++++-
.../hbase/ii/InvertedIndexStorageQuery.java | 2 +-
.../endpoint/EndpointTupleConverter.java | 4 +-
.../endpoint/EndpointTupleIterator.java | 4 +-
.../common/coprocessor/FilterBaseTest.java | 4 +-
.../common/coprocessor/FilterEvaluateTest.java | 2 +-
43 files changed, 612 insertions(+), 436 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index 9536ef7..4a95c66 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -301,11 +301,11 @@ public class TopNCounter<T> implements Iterable<Counter<T>> {
public Iterator<Counter<T>> iterator() {
return new TopNCounterIterator();
}
-
+
/**
* Iterator from the tail (smallest) to head (biggest);
*/
- private class TopNCounterIterator implements Iterator {
+ private class TopNCounterIterator implements Iterator<Counter<T>> {
private ListNode2<Counter<T>> currentBNode;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 3c91da4..8c7d891 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -767,7 +767,7 @@ public class CubeDesc extends RootPersistentEntity {
for (MeasureDesc measure : measures) {
MeasureType aggrType = measure.getFunction().getMeasureType();
- result.addAll(aggrType.getColumnsNeedDictionary(measure));
+ result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index c8836c4..3f1576a 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -41,6 +41,12 @@
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 0891d1e..d4b2700 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -19,9 +19,11 @@
package org.apache.kylin.measure;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.basic.BasicMeasureFactory;
import org.apache.kylin.measure.hllc.HLLCAggregationFactory;
import org.apache.kylin.measure.topn.TopNMeasureFactory;
@@ -32,6 +34,8 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.Maps;
@@ -82,8 +86,8 @@ abstract public class MeasureType {
abstract public MeasureAggregator<?> newAggregator();
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- return null;
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ return Collections.emptyList();
}
/* ============================================================================
@@ -102,10 +106,33 @@ abstract public class MeasureType {
abstract public boolean needRewrite();
- abstract public Class<?> getRewriteAggregationFunctionClass();
+ abstract public Class<?> getRewriteCalciteAggrFunctionClass();
/* ============================================================================
* Storage
* ---------------------------------------------------------------------------- */
+ public void beforeStorageQuery(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ }
+
+ public boolean needAdvancedTupleFilling() {
+ return false;
+ }
+
+ public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
+ tuple.setMeasureValue(indexInTuple, measureValue);
+ }
+
+ public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo returnTupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ throw new UnsupportedOperationException();
+ }
+
+ public static interface IAdvMeasureFiller {
+
+ void reload(Object measureValue);
+
+ int getNumOfRows();
+
+ void fillTuplle(Tuple tuple, int row);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index a6b36bb..fea6ba5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -122,7 +122,7 @@ public class BasicMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
// TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index fd71c00..83fcf3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -90,7 +90,7 @@ public class HLLCMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
// TODO Auto-generated method stub
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 77319be..01582d0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -20,6 +20,7 @@ package org.apache.kylin.measure.topn;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -39,9 +40,17 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
public class TopNMeasureType extends MeasureType {
+ private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
+
private final DataType dataType;
public TopNMeasureType(DataType dataType) {
@@ -134,8 +143,8 @@ public class TopNMeasureType extends MeasureType {
}
@Override
- public List<TblColRef> getColumnsNeedDictionary(MeasureDesc measureDesc) {
- TblColRef literalCol = measureDesc.getFunction().getParameter().getColRefs().get(1);
+ public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
+ TblColRef literalCol = functionDesc.getParameter().getColRefs().get(1);
return Collections.singletonList(literalCol);
}
@@ -173,8 +182,88 @@ public class TopNMeasureType extends MeasureType {
}
@Override
- public Class<?> getRewriteAggregationFunctionClass() {
+ public Class<?> getRewriteCalciteAggrFunctionClass() {
return null;
}
+ @Override
+ public void beforeStorageQuery(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ FunctionDesc topnFunc = measureDesc.getFunction();
+ TblColRef topnLiteralCol = topnFunc.getTopNLiteralColumn();
+
+ if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false)
+ return;
+
+ if (sqlDigest.aggregations.size() != 1) {
+ throw new IllegalStateException("When query with topN, only one metrics is allowed.");
+ }
+
+ FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
+ if (origFunc.isSum() == false) {
+ throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ }
+
+ sqlDigest.aggregations = Lists.newArrayList(topnFunc);
+ sqlDigest.groupbyColumns.remove(topnLiteralCol);
+ sqlDigest.metricColumns.add(topnLiteralCol);
+ logger.info("Rewrite function " + origFunc + " to " + topnFunc);
+ }
+
+ @Override
+ public boolean needAdvancedTupleFilling() {
+ return true;
+ }
+
+ @Override
+ public void fillTupleSimply(Tuple tuple, int indexInTuple, Object measureValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ final TblColRef literalCol = getTopNLiteralColumn(function);
+ final TblColRef numericCol = getTopNNumericColumn(function);
+ final Dictionary<String> topNColDict = dictionaryMap.get(literalCol);
+ final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;
+ // for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite)
+ final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1;
+
+ return new IAdvMeasureFiller() {
+ private TopNCounter<ByteArray> topNCounter;
+ private Iterator<Counter<ByteArray>> topNCounterIterator;
+ private int expectRow = 0;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void reload(Object measureValue) {
+ this.topNCounter = (TopNCounter<ByteArray>) measureValue;
+ this.topNCounterIterator = topNCounter.iterator();
+ }
+
+ @Override
+ public int getNumOfRows() {
+ return topNCounter.size();
+ }
+
+ @Override
+ public void fillTuplle(Tuple tuple, int row) {
+ if (expectRow++ != row)
+ throw new IllegalStateException();
+
+ Counter<ByteArray> counter = topNCounterIterator.next();
+ int key = BytesUtil.readUnsigned(counter.getItem().array(), 0, counter.getItem().array().length);
+ String colValue = topNColDict.getValueFromId(key);
+ tuple.setDimensionValue(literalTupleIdx, colValue);
+ tuple.setMeasureValue(numericTupleIdx, counter.getCount());
+ }
+ };
+ }
+
+ private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) {
+ return functionDesc.getParameter().getColRefs().get(0);
+ }
+
+ private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) {
+ return functionDesc.getParameter().getColRefs().get(1);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 17debda..104f3c7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -75,7 +75,7 @@ public class FunctionDesc {
parameter.setColRefs(colRefs);
// make sure sum/max/min returns the exact type as its input
- if (isSum() || isMax() || isMin() && (colRefs.size() > 0)) {
+ if ((isSum() || isMax() || isMin()) && colRefs.size() > 0) {
setReturnType(colRefs.get(0).getDatatype());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
new file mode 100644
index 0000000..c79e1f8
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tuple;
+
+import java.math.BigDecimal;
+import java.util.Date;
+import java.util.List;
+
+import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.datatype.LongMutable;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * @author xjiang
+ */
+public class Tuple implements ITuple {
+
+ @IgnoreSizeOf
+ private final TupleInfo info;
+ private final Object[] values;
+
+ public Tuple(TupleInfo info) {
+ this.info = info;
+ this.values = new Object[info.size()];
+ }
+
+ public List<String> getAllFields() {
+ return info.getAllFields();
+ }
+
+ public List<TblColRef> getAllColumns() {
+ return info.getAllColumns();
+ }
+
+ public Object[] getAllValues() {
+ return values;
+ }
+
+ @Override
+ public ITuple makeCopy() {
+ Tuple ret = new Tuple(this.info);
+ for (int i = 0; i < this.values.length; ++i) {
+ if (this.values[i] == null) {
+ ret.values[i] = null;
+ } else if (this.values[i] instanceof HyperLogLogPlusCounter) {
+ ret.values[i] = new HyperLogLogPlusCounter((HyperLogLogPlusCounter) this.values[i]);
+ } else if (this.values[i] instanceof TopNCounter) {
+ ret.values[i] = null;
+ } else {
+ ret.values[i] = this.values[i];
+ }
+ }
+ return ret;
+ }
+
+ public TupleInfo getInfo() {
+ return info;
+ }
+
+ public String getFieldName(TblColRef col) {
+ return info.getFieldName(col);
+ }
+
+ public TblColRef getFieldColumn(String fieldName) {
+ return info.getColumn(fieldName);
+ }
+
+ public Object getValue(String fieldName) {
+ int index = info.getFieldIndex(fieldName);
+ return values[index];
+ }
+
+ public Object getValue(TblColRef col) {
+ int index = info.getColumnIndex(col);
+ return values[index];
+ }
+
+ public String getDataTypeName(int idx) {
+ return info.getDataTypeName(idx);
+ }
+
+ public void setDimensionValue(String fieldName, String fieldValue) {
+ setDimensionValue(info.getFieldIndex(fieldName), fieldValue);
+ }
+
+ public void setDimensionValue(int idx, String fieldValue) {
+ Object objectValue = convertOptiqCellValue(fieldValue, getDataTypeName(idx));
+ values[idx] = objectValue;
+ }
+
+ public void setMeasureValue(String fieldName, Object fieldValue) {
+ setMeasureValue(info.getFieldIndex(fieldName), fieldValue);
+ }
+
+ public void setMeasureValue(int idx, Object fieldValue) {
+ fieldValue = convertWritableToJava(fieldValue);
+
+ String dataType = getDataTypeName(idx);
+ // special handling for BigDecimal, allow double be aggregated as
+ // BigDecimal during cube build for best precision
+ if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
+ fieldValue = ((BigDecimal) fieldValue).doubleValue();
+ } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) {
+ fieldValue = ((Number) fieldValue).intValue();
+ } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
+ fieldValue = ((BigDecimal) fieldValue).floatValue();
+ }
+ values[idx] = fieldValue;
+ }
+
+ private Object convertWritableToJava(Object o) {
+ if (o instanceof LongMutable)
+ o = ((LongMutable) o).get();
+ else if (o instanceof DoubleMutable)
+ o = ((DoubleMutable) o).get();
+ return o;
+ }
+
+ public boolean hasColumn(TblColRef column) {
+ return info.hasColumn(column);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (String field : info.getAllFields()) {
+ sb.append(field);
+ sb.append("=");
+ sb.append(getValue(field));
+ sb.append(",");
+ }
+ return sb.toString();
+ }
+
+ public static long epicDaysToMillis(int days) {
+ return 1L * days * (1000 * 3600 * 24);
+ }
+
+ public static int dateToEpicDays(String strValue) {
+ Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
+ long millis = dateValue.getTime();
+ return (int) (millis / (1000 * 3600 * 24));
+ }
+
+ public static long getTs(ITuple row, TblColRef partitionCol) {
+ //ts column type differentiate
+ if (partitionCol.getDatatype().equals("date")) {
+ return Tuple.epicDaysToMillis(Integer.valueOf(row.getValue(partitionCol).toString()));
+ } else {
+ return Long.valueOf(row.getValue(partitionCol).toString());
+ }
+ }
+
+ public static Object convertOptiqCellValue(String strValue, String dataTypeName) {
+ if (strValue == null)
+ return null;
+
+ if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string"))
+ return null;
+
+ // TODO use data type enum instead of string comparison
+ if ("date".equals(dataTypeName)) {
+ // convert epoch time
+ return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
+ } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+ return Long.valueOf(DateFormat.stringToMillis(strValue));
+ } else if ("tinyint".equals(dataTypeName)) {
+ return Byte.valueOf(strValue);
+ } else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
+ return Short.valueOf(strValue);
+ } else if ("integer".equals(dataTypeName)) {
+ return Integer.valueOf(strValue);
+ } else if ("long".equals(dataTypeName) || "bigint".equals(dataTypeName)) {
+ return Long.valueOf(strValue);
+ } else if ("double".equals(dataTypeName)) {
+ return Double.valueOf(strValue);
+ } else if ("decimal".equals(dataTypeName)) {
+ return new BigDecimal(strValue);
+ } else if ("float".equals(dataTypeName)) {
+ return Float.valueOf(strValue);
+ } else if ("boolean".equals(dataTypeName)) {
+ return Boolean.valueOf(strValue);
+ } else {
+ return strValue;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
new file mode 100644
index 0000000..f7c3b57
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tuple;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class TupleInfo {
+
+ private final Map<String, Integer> fieldMap;
+ private final Map<TblColRef, Integer> columnMap;
+
+ private final List<String> fields;
+ private final List<TblColRef> columns;
+ private final List<String> dataTypeNames;
+
+ public TupleInfo() {
+ fieldMap = new HashMap<String, Integer>();
+ columnMap = new HashMap<TblColRef, Integer>();
+ fields = new ArrayList<String>();
+ columns = new ArrayList<TblColRef>();
+ dataTypeNames = new ArrayList<String>();
+ }
+
+ public TblColRef getColumn(String fieldName) {
+ int idx = getFieldIndex(fieldName);
+ return columns.get(idx);
+ }
+
+ public int getColumnIndex(TblColRef col) {
+ return columnMap.get(col);
+ }
+
+ public String getDataTypeName(int index) {
+ return dataTypeNames.get(index);
+ }
+
+ public int getFieldIndex(String fieldName) {
+ return fieldMap.get(fieldName);
+ }
+
+ public boolean hasField(String fieldName) {
+ return fieldMap.containsKey(fieldName);
+ }
+
+ public String getFieldName(TblColRef col) {
+ int idx = columnMap.get(col);
+ return fields.get(idx);
+ }
+
+ public boolean hasColumn(TblColRef col) {
+ return columnMap.containsKey(col);
+ }
+
+ public void setField(String fieldName, TblColRef col, int index) {
+ fieldMap.put(fieldName, index);
+
+ if (col != null)
+ columnMap.put(col, index);
+
+ if (fields.size() > index)
+ fields.set(index, fieldName);
+ else
+ fields.add(index, fieldName);
+
+ if (columns.size() > index)
+ columns.set(index, col);
+ else
+ columns.add(index, col);
+
+ if (dataTypeNames.size() > index)
+ dataTypeNames.set(index, col.getType().getName());
+ else
+ dataTypeNames.add(index, col.getType().getName());
+ }
+
+ public List<String> getAllFields() {
+ return fields;
+ }
+
+ public List<TblColRef> getAllColumns() {
+ return columns;
+ }
+
+ public int size() {
+ return fields.size();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index e17d13f..ad440da 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -41,13 +41,6 @@
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>net.sf.ehcache</groupId>
- <artifactId>ehcache</artifactId>
- <version>2.8.1</version>
- </dependency>
-
-
<!-- Env & Test -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
index 9ad0af9..6b53b5b 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/IStorageQuery.java
@@ -20,7 +20,7 @@ package org.apache.kylin.storage;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.TupleInfo;
/**
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
index e1d5ee3..1481754 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedDynamicQuery.java
@@ -14,9 +14,9 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
import org.apache.kylin.metadata.tuple.TeeTupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
index af1b34c..57f33f6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/CacheFledgedStaticQuery.java
@@ -10,9 +10,9 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
import org.apache.kylin.metadata.tuple.TeeTupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
index 1ad7229..74def9e 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/cache/StreamSQLResult.java
@@ -9,7 +9,7 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.util.RangeUtil;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.Tuple;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
index 8911c0e..0107434 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageQuery.java
@@ -23,10 +23,10 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageFactory;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
deleted file mode 100644
index 3bca687..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.tuple;
-
-import java.math.BigDecimal;
-import java.util.Date;
-import java.util.List;
-
-import net.sf.ehcache.pool.sizeof.annotations.IgnoreSizeOf;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-
-/**
- * @author xjiang
- */
-public class Tuple implements ITuple {
-
- @IgnoreSizeOf
- private final TupleInfo info;
- private final Object[] values;
-
- public Tuple(TupleInfo info) {
- this.info = info;
- this.values = new Object[info.size()];
- }
-
- public List<String> getAllFields() {
- return info.getAllFields();
- }
-
- public List<TblColRef> getAllColumns() {
- return info.getAllColumns();
- }
-
- public Object[] getAllValues() {
- return values;
- }
-
- @Override
- public ITuple makeCopy() {
- Tuple ret = new Tuple(this.info);
- for (int i = 0; i < this.values.length; ++i) {
- if (this.values[i] == null) {
- ret.values[i] = null;
- } else if (this.values[i] instanceof HyperLogLogPlusCounter) {
- ret.values[i] = new HyperLogLogPlusCounter((HyperLogLogPlusCounter) this.values[i]);
- } else if (this.values[i] instanceof TopNCounter) {
- ret.values[i] = null;
- } else {
- ret.values[i] = this.values[i];
- }
- }
- return ret;
- }
-
- public TupleInfo getInfo() {
- return info;
- }
-
- public String getFieldName(TblColRef col) {
- return info.getFieldName(col);
- }
-
- public TblColRef getFieldColumn(String fieldName) {
- return info.getColumn(fieldName);
- }
-
- public Object getValue(String fieldName) {
- int index = info.getFieldIndex(fieldName);
- return values[index];
- }
-
- public Object getValue(TblColRef col) {
- int index = info.getColumnIndex(col);
- return values[index];
- }
-
- public String getDataTypeName(int idx) {
- return info.getDataTypeName(idx);
- }
-
- public void setDimensionValue(String fieldName, String fieldValue) {
- setDimensionValue(info.getFieldIndex(fieldName), fieldValue);
- }
-
- public void setDimensionValue(int idx, String fieldValue) {
- Object objectValue = convertOptiqCellValue(fieldValue, getDataTypeName(idx));
- values[idx] = objectValue;
- }
-
- public void setMeasureValue(String fieldName, Object fieldValue) {
- setMeasureValue(info.getFieldIndex(fieldName), fieldValue);
- }
-
- public void setMeasureValue(int idx, Object fieldValue) {
- fieldValue = convertWritableToJava(fieldValue);
-
- String dataType = getDataTypeName(idx);
- // special handling for BigDecimal, allow double be aggregated as
- // BigDecimal during cube build for best precision
- if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
- fieldValue = ((BigDecimal) fieldValue).doubleValue();
- } else if ("integer".equals(dataType) && !(fieldValue instanceof Integer)) {
- fieldValue = ((Number) fieldValue).intValue();
- } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
- fieldValue = ((BigDecimal) fieldValue).floatValue();
- }
- values[idx] = fieldValue;
- }
-
- private Object convertWritableToJava(Object o) {
- if (o instanceof LongMutable)
- o = ((LongMutable) o).get();
- else if (o instanceof DoubleMutable)
- o = ((DoubleMutable) o).get();
- return o;
- }
-
- public boolean hasColumn(TblColRef column) {
- return info.hasColumn(column);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (String field : info.getAllFields()) {
- sb.append(field);
- sb.append("=");
- sb.append(getValue(field));
- sb.append(",");
- }
- return sb.toString();
- }
-
- public static long epicDaysToMillis(int days) {
- return 1L * days * (1000 * 3600 * 24);
- }
-
- public static int dateToEpicDays(String strValue) {
- Date dateValue = DateFormat.stringToDate(strValue); // NOTE: forces GMT timezone
- long millis = dateValue.getTime();
- return (int) (millis / (1000 * 3600 * 24));
- }
-
- public static long getTs(ITuple row, TblColRef partitionCol) {
- //ts column type differentiate
- if (partitionCol.getDatatype().equals("date")) {
- return Tuple.epicDaysToMillis(Integer.valueOf(row.getValue(partitionCol).toString()));
- } else {
- return Long.valueOf(row.getValue(partitionCol).toString());
- }
- }
-
- public static Object convertOptiqCellValue(String strValue, String dataTypeName) {
- if (strValue == null)
- return null;
-
- if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string"))
- return null;
-
- // TODO use data type enum instead of string comparison
- if ("date".equals(dataTypeName)) {
- // convert epoch time
- return dateToEpicDays(strValue);// Optiq expects Integer instead of Long. by honma
- } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
- return Long.valueOf(DateFormat.stringToMillis(strValue));
- } else if ("tinyint".equals(dataTypeName)) {
- return Byte.valueOf(strValue);
- } else if ("short".equals(dataTypeName) || "smallint".equals(dataTypeName)) {
- return Short.valueOf(strValue);
- } else if ("integer".equals(dataTypeName)) {
- return Integer.valueOf(strValue);
- } else if ("long".equals(dataTypeName) || "bigint".equals(dataTypeName)) {
- return Long.valueOf(strValue);
- } else if ("double".equals(dataTypeName)) {
- return Double.valueOf(strValue);
- } else if ("decimal".equals(dataTypeName)) {
- return new BigDecimal(strValue);
- } else if ("float".equals(dataTypeName)) {
- return Float.valueOf(strValue);
- } else if ("boolean".equals(dataTypeName)) {
- return Boolean.valueOf(strValue);
- } else {
- return strValue;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java b/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
deleted file mode 100644
index 6f50b6d..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/tuple/TupleInfo.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.tuple;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- *
- * @author xjiang
- *
- */
-public class TupleInfo {
-
- private final Map<String, Integer> fieldMap;
- private final Map<TblColRef, Integer> columnMap;
-
- private final List<String> fields;
- private final List<TblColRef> columns;
- private final List<String> dataTypeNames;
-
- public TupleInfo() {
- fieldMap = new HashMap<String, Integer>();
- columnMap = new HashMap<TblColRef, Integer>();
- fields = new ArrayList<String>();
- columns = new ArrayList<TblColRef>();
- dataTypeNames = new ArrayList<String>();
- }
-
- public TblColRef getColumn(String fieldName) {
- int idx = getFieldIndex(fieldName);
- return columns.get(idx);
- }
-
- public int getColumnIndex(TblColRef col) {
- return columnMap.get(col);
- }
-
- public String getDataTypeName(int index) {
- return dataTypeNames.get(index);
- }
-
- public int getFieldIndex(String fieldName) {
- return fieldMap.get(fieldName);
- }
-
- public boolean hasField(String fieldName) {
- return fieldMap.containsKey(fieldName);
- }
-
- public String getFieldName(TblColRef col) {
- int idx = columnMap.get(col);
- return fields.get(idx);
- }
-
- public boolean hasColumn(TblColRef col) {
- return columnMap.containsKey(col);
- }
-
- public void setField(String fieldName, TblColRef col, int index) {
- fieldMap.put(fieldName, index);
-
- if (col != null)
- columnMap.put(col, index);
-
- if (fields.size() > index)
- fields.set(index, fieldName);
- else
- fields.add(index, fieldName);
-
- if (columns.size() > index)
- columns.set(index, col);
- else
- columns.add(index, col);
-
- if (dataTypeNames.size() > index)
- dataTypeNames.set(index, col.getType().getName());
- else
- dataTypeNames.add(index, col.getType().getName());
- }
-
- public List<String> getAllFields() {
- return fields;
- }
-
- public List<TblColRef> getAllColumns() {
- return columns;
- }
-
- public int size() {
- return fields.size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
index 1011b1a..53e5f5b 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
@@ -16,10 +16,10 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
index 09e289b..b74432d 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
@@ -16,10 +16,10 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
index 2898f93..ab25ee2 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
@@ -13,7 +13,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
index 654f1b5..b1527e6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidFromStorageMapper.java
@@ -123,7 +123,7 @@ public class MergeCuboidFromStorageMapper extends KylinMapper<Object, Object, By
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
- if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index b3f69ee..cf5049a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -125,7 +125,7 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
for (int i = 0; i < measureDescs.size(); i++) {
MeasureDesc measureDesc = measureDescs.get(i);
MeasureType measureType = measureDesc.getFunction().getMeasureType();
- if (measureType.getColumnsNeedDictionary(measureDesc).isEmpty() == false) {
+ if (measureType.getColumnsNeedDictionary(measureDesc.getFunction()).isEmpty() == false) {
dictMeasures.add(new Pair<Integer, MeasureIngester>(i, measureType.newIngester()));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 31ab924..82dc06b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,7 +75,6 @@
<!-- Spark -->
<spark.version>1.3.0</spark.version>
-
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.6.4</slf4j.version>
@@ -89,6 +88,7 @@
<extendedset.version>1.3.4</extendedset.version>
<jetty.version>9.2.7.v20150116</jetty.version>
<kryo.version>3.0.3</kryo.version>
+ <ehcache.version>2.8.1</ehcache.version>
<!-- REST Service -->
<spring.framework.version>3.1.2.RELEASE</spring.framework.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 9e59f93..1c2bc2d 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -30,10 +30,10 @@ import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPTable;
import org.apache.kylin.storage.hybrid.HybridInstance;
-import org.apache.kylin.storage.tuple.Tuple;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 9aa70ca..cebfd12 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -73,7 +73,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AGGR_FUNC_MAP.put("$SUM0", "SUM");
AGGR_FUNC_MAP.put("COUNT", "COUNT");
AGGR_FUNC_MAP.put("COUNT_DISTINCT", "COUNT_DISTINCT");
- AGGR_FUNC_MAP.put("HLL_COUNT", "COUNT_DISTINCT");
AGGR_FUNC_MAP.put("MAX", "MAX");
AGGR_FUNC_MAP.put("MIN", "MIN");
}
@@ -302,8 +301,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
SqlAggFunction newAgg = aggCall.getAggregation();
if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
- } else if (func.getMeasureType().getRewriteAggregationFunctionClass() != null) {
- newAgg = createCustomAggFunction(fieldType, func.getMeasureType().getRewriteAggregationFunctionClass());
+ } else if (func.getMeasureType().getRewriteCalciteAggrFunctionClass() != null) {
+ newAgg = createCustomAggFunction(fieldType, func.getMeasureType().getRewriteCalciteAggrFunctionClass());
}
// rebuild aggregate call
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 6865457..6de1790 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Lists;
+
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -36,9 +37,9 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.query.schema.OLAPSchema;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
/**
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index f2f9e32..304bb73 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -206,7 +206,7 @@
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
- <version>2.8.1</version>
+ <version>${ehcache.version}</version>
</dependency>
<dependency>
<groupId>cglib</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
index e781e00..b6cc6d0 100644
--- a/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/StorageCacheTest.java
@@ -24,12 +24,12 @@ import net.sf.ehcache.Element;
import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.AbstractCacheFledgedQuery;
import org.apache.kylin.storage.cache.CacheFledgedStaticQuery;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
index a8b1d02..16748a6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTopNTupleIterator.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 22f7017..4acd8f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -42,13 +42,13 @@ import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.storage.hbase.cube.v1.filter.FuzzyRowFilterV2;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 84db027..063e254 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -59,6 +59,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -67,7 +68,6 @@ import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.ColumnValueRange;
import org.apache.kylin.storage.translate.DerivedFilterTranslator;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index a5f7c74..a669e1b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -23,9 +23,9 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index 0983689..3f92cb0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -33,11 +33,11 @@ import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.apache.kylin.storage.tuple.TupleInfo;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index ca4c8aa..53b690c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -15,6 +15,7 @@ import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
@@ -26,10 +27,10 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.translate.DerivedFilterTranslator;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +53,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
// check whether this is a TopN query
- checkAndRewriteTopN(sqlDigest);
+ notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
TupleFilter filter = sqlDigest.filter;
@@ -396,33 +397,10 @@ public class CubeStorageQuery implements ICachableStorageQuery {
return false;
}
- private void checkAndRewriteTopN(SQLDigest sqlDigest) {
- FunctionDesc topnFunc = null;
- TblColRef topnLiteralCol = null;
+ private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc func = measure.getFunction();
- if (func.isTopN() && sqlDigest.groupbyColumns.contains(func.getTopNLiteralColumn())) {
- topnFunc = func;
- topnLiteralCol = func.getTopNLiteralColumn();
- }
- }
-
- // if TopN is not involved
- if (topnFunc == null)
- return;
-
- if (sqlDigest.aggregations.size() != 1) {
- throw new IllegalStateException("When query with topN, only one metrics is allowed.");
- }
-
- FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
- if (origFunc.isSum() == false) {
- throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ MeasureType measureType = measure.getFunction().getMeasureType();
+ measureType.beforeStorageQuery(measure, sqlDigest);
}
-
- sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.remove(topnLiteralCol);
- sqlDigest.metricColumns.add(topnLiteralCol);
- logger.info("Rewrite function " + origFunc + " to " + topnFunc);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
index be523d7..3653af1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeTupleConverter.java
@@ -37,12 +37,15 @@ import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* convert GTRecord to tuple
@@ -56,7 +59,12 @@ public class CubeTupleConverter {
final int[] gtColIdx;
final int[] tupleIdx;
- final Object[] tmpValues;
+ final Object[] gtValues;
+ final MeasureType[] measureTypes;
+
+ final List<IAdvMeasureFiller> advMeasureFillers;
+ final List<Integer> advMeasureIndexInGTValues;
+
final int nSelectedDims;
final TblColRef topNCol;
int topNColTupleIdx;
@@ -77,8 +85,14 @@ public class CubeTupleConverter {
nSelectedDims = selectedDimensions.size();
gtColIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
- tmpValues = new Object[selectedDimensions.size() + selectedMetrics.size()];
-
+ gtValues = new Object[selectedDimensions.size() + selectedMetrics.size()];
+
+ // measure types don't have this many, but aligned length make programming easier
+ measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];
+
+ advMeasureFillers = Lists.newArrayListWithCapacity(1);
+ advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);
+
int iii = 0;
// pre-calculate dimension index mapping to tuple
@@ -92,6 +106,7 @@ public class CubeTupleConverter {
for (FunctionDesc metric : selectedMetrics) {
int i = mapping.getIndexOf(metric);
gtColIdx[iii] = i;
+
if (metric.needRewrite()) {
String rewriteFieldName = metric.getRewriteFieldName();
tupleIdx[iii] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
@@ -101,6 +116,16 @@ public class CubeTupleConverter {
TblColRef col = metric.getParameter().getColRefs().get(0);
tupleIdx[iii] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
}
+
+ MeasureType measureType = metric.getMeasureType();
+ if (measureType.needAdvancedTupleFilling()) {
+ Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
+ advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
+ advMeasureIndexInGTValues.add(iii);
+ } else {
+ measureTypes[iii] = measureType;
+ }
+
iii++;
}
@@ -125,29 +150,49 @@ public class CubeTupleConverter {
}
}
- public void translateResult(GTRecord record, Tuple tuple) {
+ // load only needed dictionaries
+ private Map<TblColRef, Dictionary<String>> buildDictionaryMap(List<TblColRef> columnsNeedDictionary) {
+ Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
+ for (TblColRef col : columnsNeedDictionary) {
+ result.put(col, cubeSeg.getDictionary(col));
+ }
+ return result;
+ }
+
+ public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple) {
- record.getValues(gtColIdx, tmpValues);
+ record.getValues(gtColIdx, gtValues);
// dimensions
for (int i = 0; i < nSelectedDims; i++) {
int ti = tupleIdx[i];
if (ti >= 0) {
- tuple.setDimensionValue(ti, toString(tmpValues[i]));
+ tuple.setDimensionValue(ti, toString(gtValues[i]));
}
}
// measures
for (int i = nSelectedDims; i < gtColIdx.length; i++) {
int ti = tupleIdx[i];
- if (ti >= 0) {
- tuple.setMeasureValue(ti, tmpValues[i]);
+ if (ti >= 0 && measureTypes[i] != null) {
+ measureTypes[i].fillTupleSimply(tuple, ti, gtValues[i]);
}
}
// derived
for (IDerivedColumnFiller filler : derivedColFillers) {
- filler.fillDerivedColumns(tmpValues, tuple);
+ filler.fillDerivedColumns(gtValues, tuple);
+ }
+
+ // advanced measure filling, due to possible row split, will complete at caller side
+ if (advMeasureFillers.isEmpty()) {
+ return null;
+ } else {
+ for (int i = 0; i < advMeasureFillers.size(); i++) {
+ Object measureValue = gtValues[advMeasureIndexInGTValues.get(i)];
+ advMeasureFillers.get(i).reload(measureValue);
+ }
+ return advMeasureFillers;
}
}
@@ -157,15 +202,16 @@ public class CubeTupleConverter {
assert (topNCounterObj instanceof TopNCounter);
return new TopNCounterTupleIterator(tuple, (TopNCounter) topNCounterObj);
}
+
private interface IDerivedColumnFiller {
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple);
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple);
}
private IDerivedColumnFiller newDerivedColumnFiller(TblColRef[] hostCols, final DeriveInfo deriveInfo) {
boolean allHostsPresent = true;
final int[] hostTmpIdx = new int[hostCols.length];
for (int i = 0; i < hostCols.length; i++) {
- hostTmpIdx[i] = indexOnTheTmpValues(hostCols[i]);
+ hostTmpIdx[i] = indexOnTheGTValues(hostCols[i]);
allHostsPresent = allHostsPresent && hostTmpIdx[i] >= 0;
}
@@ -197,9 +243,9 @@ public class CubeTupleConverter {
}
@Override
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple) {
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
for (int i = 0; i < hostTmpIdx.length; i++) {
- lookupKey.data[i] = CubeTupleConverter.toString(tmpValues[hostTmpIdx[i]]);
+ lookupKey.data[i] = CubeTupleConverter.toString(gtValues[hostTmpIdx[i]]);
}
String[] lookupRow = lookupTable.getRow(lookupKey);
@@ -223,9 +269,9 @@ public class CubeTupleConverter {
case PK_FK:
return new IDerivedColumnFiller() {
@Override
- public void fillDerivedColumns(Object[] tmpValues, Tuple tuple) {
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
// composite keys are split, so only copy [0] is enough, see CubeDesc.initDimensionColumns()
- tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(tmpValues[hostTmpIdx[0]]));
+ tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(gtValues[hostTmpIdx[0]]));
}
};
default:
@@ -233,7 +279,7 @@ public class CubeTupleConverter {
}
}
- private int indexOnTheTmpValues(TblColRef col) {
+ private int indexOnTheGTValues(TblColRef col) {
List<TblColRef> cuboidDims = cuboid.getColumns();
int cuboidIdx = cuboidDims.indexOf(col);
for (int i = 0; i < gtColIdx.length; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
index f162be6..9eeae4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTopNTupleIterator.java
@@ -3,9 +3,9 @@ package org.apache.kylin.storage.hbase.cube.v2;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import java.util.Iterator;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
index ddaafd5..5bf5e95 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/SequentialCubeTupleIterator.java
@@ -8,13 +8,14 @@ import java.util.Set;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +35,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
protected Iterator<GTRecord> curRecordIterator;
protected CubeTupleConverter curTupleConverter;
protected Tuple next;
+
+ private List<IAdvMeasureFiller> advMeasureFillers;
+ private int advMeasureRowsRemaining;
+ private int advMeasureRowIndex;
private int scanCount;
private int scanCountDelta;
@@ -53,7 +58,19 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
public boolean hasNext() {
if (next != null)
return true;
+
+ // consume any left rows from advanced measure filler
+ if (advMeasureRowsRemaining > 0) {
+ for (IAdvMeasureFiller filler : advMeasureFillers) {
+ filler.fillTuplle(tuple, advMeasureRowIndex);
+ }
+ advMeasureRowIndex++;
+ advMeasureRowsRemaining--;
+ next = tuple;
+ return true;
+ }
+ // get the next GTRecord
if (curScanner == null) {
if (scannerIterator.hasNext()) {
curScanner = scannerIterator.next();
@@ -63,18 +80,39 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
return false;
}
}
-
- if (curRecordIterator.hasNext()) {
- curTupleConverter.translateResult(curRecordIterator.next(), tuple);
- next = tuple;
- return true;
- } else {
+ if (curRecordIterator.hasNext() == false) {
close(curScanner);
curScanner = null;
curRecordIterator = null;
curTupleConverter = null;
return hasNext();
}
+
+ // now we have a GTRecord
+ GTRecord curRecord = curRecordIterator.next();
+
+ // translate into tuple
+ List<IAdvMeasureFiller> advMeasureFillers = curTupleConverter.translateResult(curRecord, tuple);
+
+ // the simple case
+ if (advMeasureFillers == null) {
+ next = tuple;
+ return true;
+ }
+
+ // advanced measure filling, like TopN, will produce multiple tuples out of one record
+ advMeasureRowsRemaining = -1;
+ for (IAdvMeasureFiller filler : advMeasureFillers) {
+ if (advMeasureRowsRemaining < 0)
+ advMeasureRowsRemaining = filler.getNumOfRows();
+ if (advMeasureRowsRemaining != filler.getNumOfRows())
+ throw new IllegalStateException();
+ }
+ if (advMeasureRowsRemaining < 0)
+ throw new IllegalStateException();
+
+ advMeasureRowIndex = 0;
+ return hasNext();
}
@Override
@@ -119,7 +157,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
logger.error("Exception when close CubeScanner", e);
}
}
-
+
public int getScanCount() {
return scanCount;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
index eef7143..5afb62d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/InvertedIndexStorageQuery.java
@@ -25,11 +25,11 @@ import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.ICachableStorageQuery;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointTupleIterator;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
index 9640af6..1073021 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleConverter.java
@@ -6,8 +6,8 @@ import org.apache.kylin.invertedindex.index.TableRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
public class EndpointTupleConverter {
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 2fd0b4f..e638386 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -48,6 +48,8 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.cache.TsConditionExtractor;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
@@ -55,8 +57,6 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636aeecf/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
index 483b8be..92e9699 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -36,8 +36,8 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
+import org.apache.kylin.metadata.tuple.Tuple;
+import org.apache.kylin.metadata.tuple.TupleInfo;
/**
* @author xjiang