You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by vi...@apache.org on 2016/08/31 07:28:40 UTC
[1/2] incubator-carbondata git commit: Fix the bug that negtive data
compress is not properly when datatype is Double
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 0d7fa0383 -> 3eec687fe
Fix the bug that negtive data compress is not properly when datatype is Double
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3b1c8345
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3b1c8345
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3b1c8345
Branch: refs/heads/master
Commit: 3b1c8345ef2fd5d2ae3bcabcca64b3dbfacb1ccc
Parents: 0d7fa03
Author: Zhangshunyu <zh...@huawei.com>
Authored: Wed Aug 31 09:34:12 2016 +0800
Committer: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 12:50:47 2016 +0530
----------------------------------------------------------------------
.../core/util/ValueCompressionUtil.java | 26 ++++++-----
.../ValueCompressionDataTypeTestCase.scala | 48 ++++++++++++++++++++
2 files changed, 63 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3b1c8345/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 33c7a72..d3448f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -78,18 +78,18 @@ public final class ValueCompressionUtil {
private static DataType getDataType(double value, int decimal, byte dataTypeSelected) {
DataType dataType = DataType.DATA_DOUBLE;
if (decimal == 0) {
- if (value < Byte.MAX_VALUE) {
+ if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
dataType = DataType.DATA_BYTE;
- } else if (value < Short.MAX_VALUE) {
+ } else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
dataType = DataType.DATA_SHORT;
- } else if (value < Integer.MAX_VALUE) {
+ } else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
dataType = DataType.DATA_INT;
- } else if (value < Long.MAX_VALUE) {
+ } else if (value <= Long.MAX_VALUE && value >= Long.MIN_VALUE) {
dataType = DataType.DATA_LONG;
}
} else {
if (dataTypeSelected == 1) {
- if (value < Float.MAX_VALUE) {
+ if (value <= Float.MAX_VALUE && value >= Float.MIN_VALUE) {
float floatValue = (float) value;
if (floatValue - value != 0) {
dataType = DataType.DATA_DOUBLE;
@@ -97,7 +97,7 @@ public final class ValueCompressionUtil {
} else {
dataType = DataType.DATA_FLOAT;
}
- } else if (value < Double.MAX_VALUE) {
+ } else if (value <= Double.MAX_VALUE && value >= Double.MIN_VALUE) {
dataType = DataType.DATA_DOUBLE;
}
}
@@ -154,28 +154,32 @@ public final class ValueCompressionUtil {
default:
break;
}
+ //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
+ //but we can't use -1 to getDatatype, we should use -10000000.
+ double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ?
+ (double) maxValue:(double) minValue;
// None Decimal
if (decimal == 0) {
- if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) > getSize(
+ if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) > getSize(
getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_DOUBLE,
getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
- } else if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) < getSize(
+ } else if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) < getSize(
getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
} else {
return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
- getDataType((double) maxValue, decimal, dataTypeSelected));
+ getDataType(absMaxValue, decimal, dataTypeSelected));
}
}
// decimal
else {
- DataType actualDataType = getDataType((double) maxValue, decimal, dataTypeSelected);
+ DataType actualDataType = getDataType(absMaxValue, decimal, dataTypeSelected);
DataType diffDataType =
getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected);
DataType maxNonDecDataType =
- getDataType(Math.pow(10, decimal) * (double) maxValue, 0, dataTypeSelected);
+ getDataType(Math.pow(10, decimal) * absMaxValue, 0, dataTypeSelected);
DataType diffNonDecDataType =
getDataType(Math.pow(10, decimal) * ((double) maxValue - (double) minValue), 0,
dataTypeSelected);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3b1c8345/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
index 5375167..93f96d8 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
@@ -81,6 +81,54 @@ class ValueCompressionDataTypeTestCase extends QueryTest with BeforeAndAfterAll
}
}
+ test("When the values of Double datatype are negative values") {
+ val tempFilePath = "./src/test/resources/temp/doubleISnegtive.csv"
+ try {
+ sql("drop table if exists doubleISnegtive")
+ sql("drop table if exists doubleISnegtive_hive")
+ sql("CREATE TABLE doubleISnegtive (name String, value double) STORED BY 'org.apache.carbondata.format'")
+ sql("CREATE TABLE doubleISnegtive_hive (name String, value double)row format delimited fields terminated by ','")
+ val data ="a,-7489.7976000000\nb,-11234567489.797\nc,-11234567489.7\nd,-1.2\ne,-2\nf,-11234567489.7976000000\ng,-11234567489.7976000000"
+ writedata(tempFilePath, data)
+ sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive options('fileheader'='name,value')")
+ sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive_hive")
+
+ checkAnswer(sql("select * from doubleISnegtive"),
+ sql("select * from doubleISnegtive_hive"))
+ } catch{
+ case ex:Exception => ex.printStackTrace()
+ assert(false)
+ } finally {
+ sql("drop table if exists doubleISnegtive")
+ sql("drop table if exists doubleISnegtive_hive")
+ deleteFile(tempFilePath)
+ }
+ }
+
+ test("When the values of Double datatype have both postive and negative values") {
+ val tempFilePath = "./src/test/resources/temp/doublePAN.csv"
+ try {
+ sql("drop table if exists doublePAN")
+ sql("drop table if exists doublePAN_hive")
+ sql("CREATE TABLE doublePAN (name String, value double) STORED BY 'org.apache.carbondata.format'")
+ sql("CREATE TABLE doublePAN_hive (name String, value double)row format delimited fields terminated by ','")
+ val data ="a,-7489.7976000000\nb,11234567489.797\nc,-11234567489.7\nd,-1.2\ne,2\nf,-11234567489.7976000000\ng,11234567489.7976000000"
+ writedata(tempFilePath, data)
+ sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN options('fileheader'='name,value')")
+ sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN_hive")
+
+ checkAnswer(sql("select * from doublePAN"),
+ sql("select * from doublePAN_hive"))
+ } catch{
+ case ex:Exception => ex.printStackTrace()
+ assert(false)
+ } finally {
+ sql("drop table if exists doublePAN")
+ sql("drop table if exists doublePAN_hive")
+ deleteFile(tempFilePath)
+ }
+ }
+
def writedata(filePath: String, data: String) = {
val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath))
dis.writeBytes(data.toString())
[2/2] incubator-carbondata git commit: [CARBONDATA-193]Fixed the bug
that negative data compression is not proper when datatype is Double. This
closes #110
Posted by vi...@apache.org.
[CARBONDATA-193]Fixed the bug that negative data compression is not proper when datatype is Double. This closes #110
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3eec687f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3eec687f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3eec687f
Branch: refs/heads/master
Commit: 3eec687fe63c263ce1c678f3a063bf0bcc1bf28c
Parents: 0d7fa03 3b1c834
Author: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Authored: Wed Aug 31 12:58:09 2016 +0530
Committer: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 12:58:09 2016 +0530
----------------------------------------------------------------------
.../core/util/ValueCompressionUtil.java | 26 ++++++-----
.../ValueCompressionDataTypeTestCase.scala | 48 ++++++++++++++++++++
2 files changed, 63 insertions(+), 11 deletions(-)
----------------------------------------------------------------------