You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/27 10:00:58 UTC
[03/13] incubator-kylin git commit: KYLIN-976 very initial
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index 18884f7..be92f73 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -2,9 +2,9 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.DataTypeSerializer;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
@SuppressWarnings({ "rawtypes", "unchecked" })
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
index 48bda9f..0e61cf2 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTCodeSystem.java
@@ -2,8 +2,8 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.MeasureAggregator;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
public interface IGTCodeSystem {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index 1a9f637..9852acc 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.DataType;
public class UnitTestSupport {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
new file mode 100644
index 0000000..27aa07c
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/aggregation/basic/BigDecimalSerializerTest.java
@@ -0,0 +1,53 @@
+package org.apache.kylin.aggregation.basic;
+
+import static org.junit.Assert.assertEquals;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.aggregation.basic.BigDecimalSerializer;
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ */
+public class BigDecimalSerializerTest {
+
+ private static BigDecimalSerializer bigDecimalSerializer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ bigDecimalSerializer = new BigDecimalSerializer(DataType.getInstance("decimal"));
+ }
+
+ @Test
+ public void testNormal() {
+ BigDecimal input = new BigDecimal("1234.1234");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testScaleOutOfRange() {
+ BigDecimal input = new BigDecimal("1234.1234567890");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input.setScale(bigDecimalSerializer.type.getScale(), BigDecimal.ROUND_HALF_EVEN), output);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testOutOfPrecision() {
+ BigDecimal input = new BigDecimal("66855344214907231736.4924");
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ bigDecimalSerializer.serialize(input, buffer);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
new file mode 100644
index 0000000..8ee7b8d
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
@@ -0,0 +1,61 @@
+package org.apache.kylin.aggregation.topn;
+
+import org.apache.kylin.aggregation.topn.TopNCounterSerializer;
+import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.model.DataType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+public class TopNCounterSerializerTest {
+
+ private static TopNCounterSerializer serializer = new TopNCounterSerializer(DataType.getInstance("topn(10)"));
+
+ @Test
+ public void testSerialization() {
+ TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
+ Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
+ for (Integer i : stream) {
+ vs.offer(new ByteArray(Bytes.toBytes(i)));
+ }
+
+ ByteBuffer out = ByteBuffer.allocate(1024);
+ serializer.serialize(vs, out);
+
+ byte[] copyBytes = new byte[out.position()];
+ System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
+
+ ByteBuffer in = ByteBuffer.wrap(copyBytes);
+ TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
+
+ Assert.assertEquals(vs.toString(), vsNew.toString());
+
+ }
+
+ @Test
+ public void testValueOf() {
+
+ TopNCounter<ByteArray> origin = new TopNCounter<ByteArray>(10);
+ ByteArray key = new ByteArray(1);
+ ByteBuffer byteBuffer = key.asBuffer();
+ BytesUtil.writeVLong(20l, byteBuffer);
+ origin.offer(key, 1.0);
+
+ byteBuffer = ByteBuffer.allocate(1024);
+ byteBuffer.putInt(1);
+ byteBuffer.putInt(20);
+ byteBuffer.putDouble(1.0);
+ TopNCounter<ByteArray> counter = serializer.valueOf(byteBuffer.array());
+
+
+ Assert.assertEquals(origin.toString(), counter.toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index fa7d611..f407977 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -23,15 +23,15 @@ import java.util.Random;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.kylin.aggregation.MeasureAggregator;
+import org.apache.kylin.aggregation.basic.BigDecimalSumAggregator;
+import org.apache.kylin.aggregation.basic.DoubleSumAggregator;
+import org.apache.kylin.aggregation.basic.LongSumAggregator;
+import org.apache.kylin.aggregation.hllc.HLLCAggregator;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.measure.BigDecimalSumAggregator;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.measure.DoubleSumAggregator;
-import org.apache.kylin.metadata.measure.HLLCAggregator;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.LongSumAggregator;
-import org.apache.kylin.metadata.measure.MeasureAggregator;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.junit.Test;
public class AggregationCacheMemSizeTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 91e7e18..f5247e2 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.Dictionary;
@@ -44,7 +45,6 @@ import org.apache.kylin.metadata.filter.ExtractTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TableDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 6395a3b..361617a 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -27,8 +27,8 @@ import java.util.BitSet;
import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.measure.LongMutable;
import org.junit.Test;
public class SimpleGridTableTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 41d75c9..a964e67 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -23,20 +23,21 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import org.apache.kylin.aggregation.basic.StringSerializer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.serializer.StringSerializer;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.Test;
import com.google.common.collect.Lists;
+
import it.uniroma3.mat.extendedset.intset.ConciseSet;
public class SimpleInvertedIndexTest {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index ad5048b..0e13d7e 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -23,7 +23,10 @@ import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import org.apache.kylin.aggregation.MeasureCodec;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.common.util.DoubleMutable;
+import org.apache.kylin.common.util.LongMutable;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
deleted file mode 100644
index 91713a4..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMaxAggregator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) < 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
deleted file mode 100644
index 6bbb6a9..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalMinAggregator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
- }
-
- @Override
- public BigDecimal getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
deleted file mode 100644
index f1c60a1..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/BigDecimalSumAggregator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.math.BigDecimal;
-
-/**
- */
-@SuppressWarnings("serial")
-public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
-
- BigDecimal sum = new BigDecimal(0);
-
- @Override
- public void reset() {
- sum = new BigDecimal(0);
- }
-
- @Override
- public void aggregate(BigDecimal value) {
- sum = sum.add(value);
- }
-
- @Override
- public BigDecimal getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessBigDecimalMemBytes();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
deleted file mode 100644
index 962cd75..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMaxAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
deleted file mode 100644
index f8c3680..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMinAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
deleted file mode 100644
index 10072a6..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleMutable.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class DoubleMutable implements Comparable<DoubleMutable>, Serializable {
-
- private double v;
-
- public DoubleMutable() {
- this(0);
- }
-
- public DoubleMutable(double v) {
- set(v);
- }
-
- public double get() {
- return v;
- }
-
- public void set(double v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof DoubleMutable)) {
- return false;
- }
- DoubleMutable other = (DoubleMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) Double.doubleToLongBits(v);
- }
-
- @Override
- public int compareTo(DoubleMutable o) {
- return (v < o.v ? -1 : (v == o.v ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Double.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
deleted file mode 100644
index 7593cac..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/DoubleSumAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
-
- DoubleMutable sum = new DoubleMutable();
-
- @Override
- public void reset() {
- sum.set(0.0);
- }
-
- @Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public DoubleMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessDoubleMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
deleted file mode 100644
index f76cf64..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCAggregator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-
-/**
- */
-@SuppressWarnings("serial")
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
-
- final int precision;
- HyperLogLogPlusCounter sum = null;
-
- public HLLCAggregator(int precision) {
- this.precision = precision;
- }
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(HyperLogLogPlusCounter value) {
- if (sum == null)
- sum = new HyperLogLogPlusCounter(value);
- else
- sum.merge(value);
- }
-
- @Override
- public HyperLogLogPlusCounter getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- // 1024 + 60 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 4 // precision
- + 8 // ref to HLLC
- + 8 // HLLC obj shell
- + 32 + (1 << precision); // HLLC internal
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
deleted file mode 100644
index e5cf2a9..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LDCAggregator.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- * Long Distinct Count
- */
-@SuppressWarnings("serial")
-public class LDCAggregator extends MeasureAggregator<LongMutable> {
-
- private static LongMutable ZERO = new LongMutable(0);
-
- private HLLCAggregator hllAgg = null;
- private LongMutable state = new LongMutable(0);
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- this.hllAgg = (HLLCAggregator) agg;
- }
-
- @Override
- public void reset() {
- }
-
- @Override
- public void aggregate(LongMutable value) {
- }
-
- @Override
- public LongMutable getState() {
- if (hllAgg == null) {
- return ZERO;
- } else {
- state.set(hllAgg.getState().getCountEstimate());
- return state;
- }
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
deleted file mode 100644
index 5b080bb..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMaxAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable max = null;
-
- @Override
- public void reset() {
- max = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return max;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
deleted file mode 100644
index a2509ac..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMinAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable min = null;
-
- @Override
- public void reset() {
- min = null;
- }
-
- @Override
- public void aggregate(LongMutable value) {
- if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
- }
-
- @Override
- public LongMutable getState() {
- return min;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
deleted file mode 100644
index 220b6a6..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongMutable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-
-@SuppressWarnings("serial")
-public class LongMutable implements Comparable<LongMutable>, Serializable {
-
- private long v;
-
- public LongMutable() {
- this(0);
- }
-
- public LongMutable(long v) {
- set(v);
- }
-
- public long get() {
- return v;
- }
-
- public void set(long v) {
- this.v = v;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof LongMutable)) {
- return false;
- }
- LongMutable other = (LongMutable) o;
- return this.v == other.v;
- }
-
- @Override
- public int hashCode() {
- return (int) v;
- }
-
- @Override
- public int compareTo(LongMutable o) {
- long thisValue = this.v;
- long thatValue = o.v;
- return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1));
- }
-
- @Override
- public String toString() {
- return Long.toString(v);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
deleted file mode 100644
index 63422f5..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/LongSumAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-/**
- */
-@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
-
- LongMutable sum = new LongMutable();
-
- @Override
- public void reset() {
- sum.set(0);
- }
-
- @Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
- }
-
- @Override
- public LongMutable getState() {
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return guessLongMemBytes();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
deleted file mode 100644
index 1b38aa5..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-
-import java.io.Serializable;
-
-/**
- */
-@SuppressWarnings("serial")
-abstract public class MeasureAggregator<V> implements Serializable {
-
- public static MeasureAggregator<?> create(String funcName, String returnType) {
- if (FunctionDesc.FUNC_SUM.equalsIgnoreCase(funcName) || FunctionDesc.FUNC_COUNT.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongSumAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalSumAggregator();
- else if (isDouble(returnType))
- return new DoubleSumAggregator();
- } else if (FunctionDesc.FUNC_COUNT_DISTINCT.equalsIgnoreCase(funcName)) {
- DataType hllcType = DataType.getInstance(returnType);
- if (hllcType.isHLLC())
- return new HLLCAggregator(hllcType.getPrecision());
- else
- return new LDCAggregator();
- } else if (FunctionDesc.FUNC_MAX.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMaxAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMaxAggregator();
- else if (isDouble(returnType))
- return new DoubleMaxAggregator();
- } else if (FunctionDesc.FUNC_MIN.equalsIgnoreCase(funcName)) {
- if (isInteger(returnType))
- return new LongMinAggregator();
- else if (isBigDecimal(returnType))
- return new BigDecimalMinAggregator();
- else if (isDouble(returnType))
- return new DoubleMinAggregator();
- } else if (FunctionDesc.FUNC_TOP_N.equalsIgnoreCase(funcName)) {
- return new TopNAggregator();
- }
- throw new IllegalArgumentException("No aggregator for func '" + funcName + "' and return type '" + returnType + "'");
- }
-
- public static boolean isBigDecimal(String type) {
- return type.startsWith("decimal");
- }
-
- public static boolean isDouble(String type) {
- return "double".equalsIgnoreCase(type) || "float".equalsIgnoreCase(type) || "real".equalsIgnoreCase(type);
- }
-
- public static boolean isInteger(String type) {
- return "long".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type) || "integer".equalsIgnoreCase(type);
- }
-
- public static int guessBigDecimalMemBytes() {
- // 116 returned by AggregationCacheMemSizeTest
- return 8 // aggregator obj shell
- + 8 // ref to BigDecimal
- + 8 // BigDecimal obj shell
- + 100; // guess of BigDecimal internal
- }
-
- public static int guessDoubleMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to DoubleWritable
- + 8 // DoubleWritable obj shell
- + 8; // size of double
- */
- }
-
- public static int guessLongMemBytes() {
- // 29 to 44 returned by AggregationCacheMemSizeTest
- return 44;
- /*
- return 8 // aggregator obj shell
- + 8 // ref to LongWritable
- + 8 // LongWritable obj shell
- + 8; // size of long
- */
- }
-
- // ============================================================================
-
- @SuppressWarnings("rawtypes")
- public void setDependentAggregator(MeasureAggregator agg) {
- }
-
- abstract public void reset();
-
- abstract public void aggregate(V value);
-
- abstract public V getState();
-
- // get an estimate of memory consumption UPPER BOUND
- abstract public int getMemBytesEstimate();
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
deleted file mode 100644
index 09e553e..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureAggregators.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- */
-@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
-public class MeasureAggregators implements Serializable {
-
- private final MeasureAggregator[] aggs;
- private final int descLength;
-
- public MeasureAggregators(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureAggregators(MeasureDesc... measureDescs) {
- descLength = measureDescs.length;
- aggs = new MeasureAggregator[descLength];
-
- Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
- for (int i = 0; i < descLength; i++) {
- FunctionDesc func = measureDescs[i].getFunction();
- aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
- measureIndexMap.put(measureDescs[i].getName(), i);
- }
- // fill back dependent aggregator
- for (int i = 0; i < descLength; i++) {
- String depMsrRef = measureDescs[i].getDependentMeasureRef();
- if (depMsrRef != null) {
- int index = measureIndexMap.get(depMsrRef);
- aggs[i].setDependentAggregator(aggs[index]);
- }
- }
- }
-
- public void reset() {
- for (int i = 0; i < aggs.length; i++) {
- aggs[i].reset();
- }
- }
-
- public void aggregate(Object[] values) {
- assert values.length == descLength;
-
- for (int i = 0; i < descLength; i++) {
- aggs[i].aggregate(values[i]);
- }
- }
-
- public void collectStates(Object[] states) {
- for (int i = 0; i < descLength; i++) {
- states[i] = aggs[i].getState();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
deleted file mode 100644
index 296290a..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/MeasureCodec.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
-import org.apache.kylin.metadata.measure.serializer.DataTypeSerializer;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MeasureCodec {
-
- int nMeasures;
- DataTypeSerializer[] serializers;
-
- public MeasureCodec(Collection<MeasureDesc> measureDescs) {
- this((MeasureDesc[]) measureDescs.toArray(new MeasureDesc[measureDescs.size()]));
- }
-
- public MeasureCodec(MeasureDesc... measureDescs) {
- String[] dataTypes = new String[measureDescs.length];
- for (int i = 0; i < dataTypes.length; i++) {
- dataTypes[i] = measureDescs[i].getFunction().getReturnType();
- }
- init(dataTypes);
- }
-
- public MeasureCodec(String... dataTypes) {
- init(dataTypes);
- }
-
- private void init(String[] dataTypes) {
- nMeasures = dataTypes.length;
- serializers = new DataTypeSerializer[nMeasures];
-
- for (int i = 0; i < nMeasures; i++) {
- serializers[i] = DataTypeSerializer.create(dataTypes[i]);
- }
- }
-
- public DataTypeSerializer getSerializer(int idx) {
- return serializers[idx];
- }
-
- public void decode(ByteBuffer buf, Object[] result) {
- assert result.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- result[i] = serializers[i].deserialize(buf);
- }
- }
-
- public void encode(Object[] values, ByteBuffer out) {
- assert values.length == nMeasures;
- for (int i = 0; i < nMeasures; i++) {
- serializers[i].serialize(values[i], out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
deleted file mode 100644
index 9c21f7b..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/TopNAggregator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.common.topn.Counter;
-import org.apache.kylin.common.topn.TopNCounter;
-import org.apache.kylin.common.util.ByteArray;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- *
- */
-@SuppressWarnings("serial")
-public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
-
- int capacity = 0;
- TopNCounter<ByteArray> sum = null;
- Map<ByteArray, Double> sanityCheckMap;
-
- @Override
- public void reset() {
- sum = null;
- }
-
- @Override
- public void aggregate(TopNCounter<ByteArray> value) {
- if (sum == null) {
- capacity = value.getCapacity();
- sum = new TopNCounter<>(capacity);
- sanityCheckMap = Maps.newHashMap();
- }
- sum.merge(value);
- }
-
- @Override
- public TopNCounter<ByteArray> getState() {
-
- //sum.retain(capacity);
- return sum;
- }
-
- @Override
- public int getMemBytesEstimate() {
- return 8 * capacity / 4;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
deleted file mode 100644
index a0af6a2..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
-
- private DataType type;
- private int presision;
- private HyperLogLogPlusCounter current;
-
- public FixedHLLCodec(DataType type) {
- this.type = type;
- this.presision = type.getPrecision();
- this.current = new HyperLogLogPlusCounter(this.presision);
- }
-
- @Override
- public int getLength() {
- return 1 << presision;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(String value) {
- current.clear();
- if (value == null)
- current.add("__nUlL__");
- else
- current.add(value.getBytes());
- return current;
- }
-
- @Override
- public Object getValue() {
- return current;
- }
-
- @Override
- public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- return current;
- }
-
- @Override
- public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
- v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
- }
-
- @Override
- public HyperLogLogPlusCounter read(ByteBuffer buffer) {
- current.readRegistersArray(buffer);
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
deleted file mode 100644
index 54c4eb8..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-
-abstract public class FixedLenMeasureCodec<T> {
-
- public static FixedLenMeasureCodec<?> get(DataType type) {
- if (type.isHLLC()) {
- return new FixedHLLCodec(type);
- } else {
- return new FixedPointLongCodec(type);
- }
- }
-
- abstract public int getLength();
-
- abstract public DataType getDataType();
-
- abstract public T valueOf(String value);
-
- abstract public Object getValue();
-
- abstract public T read(byte[] buf, int offset);
-
- abstract public void write(T v, byte[] buf, int offset);
-
- abstract public T read(ByteBuffer buffer);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
deleted file mode 100644
index 37eb0be..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.fixedlen;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-import java.nio.ByteBuffer;
-
-public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> {
-
- private static final int SIZE = 8;
- // number of digits after decimal point
- int scale;
- DataType type;
- // avoid massive object creation
- LongMutable current = new LongMutable();
-
- public FixedPointLongCodec(DataType type) {
- this.type = type;
- this.scale = Math.max(0, type.getScale());
- }
-
- @Override
- public int getLength() {
- return SIZE;
- }
-
- @Override
- public DataType getDataType() {
- return type;
- }
-
- long getValueIgnoringDecimalPoint(String value) {
- int index = value.indexOf('.');
-
- if (index == 0 || index == value.length() - 1) {
- throw new RuntimeException("Bad decimal format: " + value);
- } else if (index < 0) {
- return Long.valueOf(value) * (int) Math.pow(10, scale);
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append(value.substring(0, index));
-
- //if there are more than scale digits after the decimal point, the tail will be discarded
- int end = Math.min(value.length(), index + scale + 1);
- sb.append(value.substring(index + 1, end));
- int diff = index + scale + 1 - value.length();
- //if there are less than scale digits after the decimal point, the tail will be compensated
- for (int i = 0; i < diff; i++) {
- sb.append('0');
- }
- return Long.valueOf(sb.toString());
- }
- }
-
- String restoreDecimalPoint(long value) {
- if (scale < 0) {
- throw new RuntimeException("Bad scale: " + scale + " with value: " + value);
- } else if (scale == 0) {
- return Long.toString(value);
- } else {
- return String.format("%." + scale + "f", value / (Math.pow(10, scale)));
- }
- }
-
- @Override
- public LongMutable valueOf(String value) {
- if (value == null)
- current.set(0L);
- else
- current.set(getValueIgnoringDecimalPoint(value));
- return current;
- }
-
- @Override
- public String getValue() {
- if (scale == 0)
- return current.toString();
- else
- return restoreDecimalPoint(current.get());
- }
-
- @Override
- public LongMutable read(byte[] buf, int offset) {
- current.set(BytesUtil.readLong(buf, offset, SIZE));
- return current;
- }
-
- @Override
- public void write(LongMutable v, byte[] buf, int offset) {
- BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE);
- }
-
- @Override
- public LongMutable read(ByteBuffer buffer) {
- current.set(BytesUtil.readLong(buffer, SIZE));
- return current;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
deleted file mode 100644
index d10c565..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/BigDecimalSerializer.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.model.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author yangli9
- *
- */
-public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
-
- private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
-
- final DataType type;
- final int maxLength;
-
- int avoidVerbose = 0;
-
- public BigDecimalSerializer(DataType type) {
- this.type = type;
- // see serialize(): 1 byte scale, 1 byte length, assume every 2 digits takes 1 byte
- this.maxLength = 1 + 1 + (type.getPrecision() + 1) / 2;
- }
-
- @Override
- public void serialize(BigDecimal value, ByteBuffer out) {
- if (value.scale() > type.getScale()) {
- if (avoidVerbose % 10000 == 0) {
- logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose++));
- }
- value = value.setScale(type.getScale(), BigDecimal.ROUND_HALF_EVEN);
- }
- byte[] bytes = value.unscaledValue().toByteArray();
- if (bytes.length + 2 > maxLength) {
- throw new IllegalArgumentException("'" + value + "' exceeds the expected length for type " + type);
- }
-
- BytesUtil.writeVInt(value.scale(), out);
- BytesUtil.writeVInt(bytes.length, out);
- out.put(bytes);
- }
-
- @Override
- public BigDecimal deserialize(ByteBuffer in) {
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
-
- byte[] bytes = new byte[n];
- in.get(bytes);
-
- return new BigDecimal(new BigInteger(bytes), scale);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- @SuppressWarnings("unused")
- int scale = BytesUtil.readVInt(in);
- int n = BytesUtil.readVInt(in);
- int len = in.position() - mark + n;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public BigDecimal valueOf(byte[] value) {
- if (value == null)
- return new BigDecimal(0);
- else
- return new BigDecimal(Bytes.toString(value));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
deleted file mode 100644
index d542098..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.metadata.model.DataType;
-
-import com.google.common.collect.Maps;
-
-/**
- * @author yangli9
- *
- * Note: the implementations MUST be thread-safe.
- *
- */
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
-
- final static Map<String, Class<?>> implementations;
- static {
- HashMap<String, Class<?>> impl = Maps.newHashMap();
- impl.put("varchar", StringSerializer.class);
- impl.put("decimal", BigDecimalSerializer.class);
- impl.put("double", DoubleSerializer.class);
- impl.put("float", DoubleSerializer.class);
- impl.put("bigint", LongSerializer.class);
- impl.put("long", LongSerializer.class);
- impl.put("integer", LongSerializer.class);
- impl.put("int", LongSerializer.class);
- impl.put("smallint", LongSerializer.class);
- impl.put("date", DateTimeSerializer.class);
- impl.put("datetime", DateTimeSerializer.class);
- impl.put("timestamp", DateTimeSerializer.class);
- implementations = Collections.unmodifiableMap(impl);
-
- }
-
- public static DataTypeSerializer<?> create(String dataType) {
- return create(DataType.getInstance(dataType));
- }
-
- public static DataTypeSerializer<?> create(DataType type) {
- if (type.isHLLC()) {
- return new HLLCSerializer(type);
- }
-
- if (type.isTopN()) {
- return new TopNCounterSerializer(type);
- }
-
- Class<?> clz = implementations.get(type.getName());
- if (clz == null)
- throw new RuntimeException("No MeasureSerializer for type " + type);
-
- try {
- return (DataTypeSerializer<?>) clz.getConstructor(DataType.class).newInstance(type);
- } catch (Exception e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** peek into buffer and return the length of serialization */
- abstract public int peekLength(ByteBuffer in);
-
- /** return the max number of bytes to the longest serialization */
- abstract public int maxLength();
-
- /** get an estimate of size in bytes of the serialized data */
- abstract public int getStorageBytesEstimate();
-
- /** convert from String to obj (string often come as byte[] in mapred) */
- abstract public T valueOf(byte[] value);
-
- /** convert from String to obj */
- public T valueOf(String value) {
- try {
- return valueOf(value.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- /** convert from obj to string */
- public String toString(T value) {
- if (value == null)
- return "NULL";
- else
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
deleted file mode 100644
index d79c382..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DateTimeSerializer.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public DateTimeSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(DateFormat.stringToMillis(Bytes.toString(value)));
- return l;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
deleted file mode 100644
index 2dc8a67..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DoubleSerializer.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.measure.DoubleMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
-
- public DoubleSerializer(DataType type) {
- }
-
- @Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
- }
-
- @Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return 8;
- }
-
- @Override
- public int maxLength() {
- return 8;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 8;
- }
-
- @Override
- public DoubleMutable valueOf(byte[] value) {
- DoubleMutable d = current();
- if (value == null)
- d.set(0d);
- else
- d.set(Double.parseDouble(Bytes.toString(value)));
- return d;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
deleted file mode 100644
index 7aed458..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/HLLCSerializer.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- * @author yangli9
- *
- */
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
-
- private int precision;
-
- public HLLCSerializer(DataType type) {
- this.precision = type.getPrecision();
- }
-
- @Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
- try {
- value.writeRegisters(out);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private HyperLogLogPlusCounter current() {
- HyperLogLogPlusCounter hllc = current.get();
- if (hllc == null) {
- hllc = new HyperLogLogPlusCounter(precision);
- current.set(hllc);
- }
- return hllc;
- }
-
- @Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- HyperLogLogPlusCounter hllc = current();
- try {
- hllc.readRegisters(in);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return hllc;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return current().peekLength(in);
- }
-
- @Override
- public int maxLength() {
- return current().maxLength();
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return current().maxLength();
- }
-
- @Override
- public HyperLogLogPlusCounter valueOf(byte[] value) {
- HyperLogLogPlusCounter hllc = current();
- hllc.clear();
- if (value == null)
- hllc.add("__nUlL__");
- else
- hllc.add(value);
- return hllc;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1218bbde/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
deleted file mode 100644
index f51a52c..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/LongSerializer.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.measure.serializer;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.model.DataType;
-
-/**
- */
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
-
- public LongSerializer(DataType type) {
- }
-
- @Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
-
- @Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
-
- BytesUtil.readVLong(in);
- int len = in.position() - mark;
-
- in.position(mark);
- return len;
- }
-
- @Override
- public int maxLength() {
- return 9; // vlong: 1 + 8
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return 5;
- }
-
- @Override
- public LongMutable valueOf(byte[] value) {
- LongMutable l = current();
- if (value == null)
- l.set(0L);
- else
- l.set(Long.parseLong(Bytes.toString(value)));
- return l;
- }
-
-}