You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/08/09 09:31:39 UTC
carbondata git commit: [HOTFIX]Fixed int overflow and comparison gone
wrong during blocklet min/max
Repository: carbondata
Updated Branches:
refs/heads/master 7158d5203 -> 8affab843
[HOTFIX]Fixed int overflow and comparison gone wrong during blocklet min/max
Problem: During calculating min/max for blocklet, it needs to calculate from all the pages. During that comparison,
it is typecasting to int and overflows, so there is a chance the negative becomes positive and positive become negative.
That's why min max of long comes wrong for bigger values.
Solution: Don't typecast directly, instead check first the negative or positive and then return.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8affab84
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8affab84
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8affab84
Branch: refs/heads/master
Commit: 8affab8433bb4dea70fbb4ea9d3abc7eaf9fd7b2
Parents: 7158d52
Author: ravipesala <ra...@gmail.com>
Authored: Tue Aug 7 21:19:36 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Aug 9 15:01:12 2018 +0530
----------------------------------------------------------------------
.../core/util/CarbonMetadataUtil.java | 16 +++++++-
.../core/util/CarbonMetadataUtilTest.java | 39 +++++++++++++-------
2 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8affab84/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 8fc648b..70443d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -368,7 +368,13 @@ public class CarbonMetadataUtil {
secondBuffer.put(second);
firstBuffer.flip();
secondBuffer.flip();
- return (int) (firstBuffer.getDouble() - secondBuffer.getDouble());
+ double compare = firstBuffer.getDouble() - secondBuffer.getDouble();
+ if (compare > 0) {
+ compare = 1;
+ } else if (compare < 0) {
+ compare = -1;
+ }
+ return (int) compare;
} else if (dataType == DataTypes.LONG || dataType == DataTypes.INT
|| dataType == DataTypes.SHORT) {
firstBuffer = ByteBuffer.allocate(8);
@@ -377,7 +383,13 @@ public class CarbonMetadataUtil {
secondBuffer.put(second);
firstBuffer.flip();
secondBuffer.flip();
- return (int) (firstBuffer.getLong() - secondBuffer.getLong());
+ long compare = firstBuffer.getLong() - secondBuffer.getLong();
+ if (compare > 0) {
+ compare = 1;
+ } else if (compare < 0) {
+ compare = -1;
+ }
+ return (int) compare;
} else if (DataTypes.isDecimal(dataType)) {
return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8affab84/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 2909dc4..14cd57a 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -17,40 +17,28 @@
package org.apache.carbondata.core.util;
+import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.page.EncodedTablePage;
-import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
-import org.apache.carbondata.core.datastore.page.key.TablePageKey;
-import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.format.BlockIndex;
-import org.apache.carbondata.format.BlockletIndex;
import org.apache.carbondata.format.BlockletInfo;
-import org.apache.carbondata.format.BlockletInfo3;
-import org.apache.carbondata.format.BlockletMinMaxIndex;
import org.apache.carbondata.format.ColumnSchema;
import org.apache.carbondata.format.DataChunk;
-import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataType;
import org.apache.carbondata.format.Encoding;
-import org.apache.carbondata.format.FileFooter3;
import org.apache.carbondata.format.IndexHeader;
import org.apache.carbondata.format.SegmentInfo;
-import mockit.Mock;
-import mockit.MockUp;
import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.TestCase.assertEquals;
-import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertFileFooterVersion3;
import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockIndexInfo;
-import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockletIndex;
import static org.apache.carbondata.core.util.CarbonMetadataUtil.getIndexHeader;
public class CarbonMetadataUtilTest {
@@ -205,4 +193,27 @@ public class CarbonMetadataUtilTest {
assertEquals(result.get(0).file_name, expected);
}
+ @Test public void testGetBlockletIndex() throws Exception {
+
+ long left = Long.MAX_VALUE;
+ long right = 100;
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ buffer.putLong(left);
+ buffer.flip();
+ byte[] l = buffer.array().clone();
+
+ buffer.rewind();
+ buffer.putLong(right);
+ buffer.flip();
+ byte[] r = buffer.array().clone();
+
+ Method method = CarbonMetadataUtil.class
+ .getDeclaredMethod("compareMeasureData", l.getClass(), r.getClass(),
+ org.apache.carbondata.core.metadata.datatype.DataType.class);
+ method.setAccessible(true);
+ int out = (int)method.invoke(method, l, r, DataTypes.LONG);
+ assertEquals(1, out);
+
+ }
+
}