You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/14 09:28:36 UTC
[kylin] 01/10: KYLIN-3364 make it consistent with hive for
BigDecimalSumAggregator dealing with null
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ec927e3f5ebfac4be04e54d0194556f71478a4f5
Author: Zhong <nj...@apache.org>
AuthorDate: Sun May 6 16:43:41 2018 +0800
KYLIN-3364 make it consistent with hive for BigDecimalSumAggregator dealing with null
---
.../apache/kylin/measure/basic/BigDecimalSumAggregator.java | 9 +++++++--
.../apache/kylin/metadata/datatype/BigDecimalSerializer.java | 10 ++++++++++
.../kylin/metadata/datatype/BigDecimalSerializerTest.java | 10 ++++++++++
.../java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java | 9 +++++----
4 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
index 9f6ffc2..fa59c54 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
@@ -27,15 +27,20 @@ import org.apache.kylin.measure.MeasureAggregator;
@SuppressWarnings("serial")
public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
- BigDecimal sum = new BigDecimal(0);
+ BigDecimal sum = null;
@Override
public void reset() {
- sum = new BigDecimal(0);
+ sum = null;
}
@Override
public void aggregate(BigDecimal value) {
+ if (value == null)
+ return;
+ if (sum == null) {
+ sum = new BigDecimal(0);
+ }
sum = sum.add(value);
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index ba1c4ff..967c00d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -46,6 +46,12 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
@Override
public void serialize(BigDecimal value, ByteBuffer out) {
+ if (value == null) {
+ BytesUtil.writeVInt(0, out);
+ BytesUtil.writeVInt(-1, out);
+ return;
+ }
+
if (value.scale() > type.getScale()) {
if (avoidVerbose++ % 10000 == 0) {
logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose));
@@ -67,6 +73,10 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
int scale = BytesUtil.readVInt(in);
int n = BytesUtil.readVInt(in);
+ if (n < 0) {
+ return null;
+ }
+
byte[] bytes = new byte[n];
in.get(bytes);
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
index 5be5806..a21b08e 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java
@@ -74,4 +74,14 @@ public class BigDecimalSerializerTest extends LocalFileMetadataTestCase {
bigDecimalSerializer.serialize(input, buffer);
}
+ @Test
+ public void testNull() {
+ BigDecimal input = null;
+ ByteBuffer buffer = ByteBuffer.allocate(256);
+ buffer.mark();
+ bigDecimalSerializer.serialize(input, buffer);
+ buffer.reset();
+ BigDecimal output = bigDecimalSerializer.deserialize(buffer);
+ assertEquals(input, output);
+ }
}
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 7616df2..f90a644 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -148,9 +148,9 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
List<Pair<Text, Text>> result = reduceDriver.run();
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0));
+ Pair<Text, Text> p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600));
+ Pair<Text, Text> p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500));
+ Pair<Text, Text> p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0));
assertEquals(3, result.size());
@@ -160,7 +160,8 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
}
private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };
+ Object[] values = new Object[] { sum == null ? null : new BigDecimal(sum), //
+ new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };
ByteBuffer buf = codec.encode(values);
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.