You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/21 18:00:13 UTC
[46/50] [abbrv] carbondata git commit: [CARBONDATA-3098] Fix for
negative exponents value giving wrong results in Float datatype
[CARBONDATA-3098] Fix for negative exponents value giving wrong results in Float datatype
Problem: When the value of exponent is a negative number then the data is incorrect due to loss of precision of Floating point values and wrong
calculation of the count of decimal points.
Solution: Handled floating point precision by converting it to double and counted the decimal count values as done in double datatype(using Big Decimal).
This closes #2918
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6e626982
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6e626982
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6e626982
Branch: refs/heads/branch-1.5
Commit: 6e6269824a52e6b2abef3c7ac52d9e458ee29f8c
Parents: e4843d7
Author: Manish Nalla <ma...@gmail.com>
Authored: Wed Nov 14 10:57:49 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Nov 21 22:43:46 2018 +0530
----------------------------------------------------------------------
.../encoding/adaptive/AdaptiveFloatingCodec.java | 14 +-------------
.../page/statistics/PrimitivePageStatsCollector.java | 14 +-------------
.../datasource/SparkCarbonDataSourceTest.scala | 15 +++++++++++++++
3 files changed, 17 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e626982/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index 49696eb..b04c9df 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -167,19 +167,7 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
@Override
public void encode(int rowId, float value) {
- if (targetDataType == DataTypes.BYTE) {
- encodedPage.putByte(rowId, (byte) (value * floatFactor));
- } else if (targetDataType == DataTypes.SHORT) {
- encodedPage.putShort(rowId, (short) (value * floatFactor));
- } else if (targetDataType == DataTypes.SHORT_INT) {
- encodedPage.putShortInt(rowId, (int) (value * floatFactor));
- } else if (targetDataType == DataTypes.INT) {
- encodedPage.putInt(rowId, (int) (value * floatFactor));
- } else if (targetDataType == DataTypes.LONG) {
- encodedPage.putLong(rowId, (long) (value * floatFactor));
- } else {
- throw new RuntimeException("internal error: " + debugInfo());
- }
+ encode(rowId, (double) value);
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e626982/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 9be5a58..e604057 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -253,19 +253,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
}
private int getDecimalCount(float value) {
- int decimalPlaces = 0;
- try {
- String strValue = Float.valueOf(Math.abs(value)).toString();
- int integerPlaces = strValue.indexOf('.');
- if (-1 != integerPlaces) {
- decimalPlaces = strValue.length() - integerPlaces - 1;
- }
- } catch (NumberFormatException e) {
- if (!Double.isInfinite(value)) {
- throw e;
- }
- }
- return decimalPlaces;
+ return getDecimalCount((double) value);
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6e626982/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
index 1e58a9e..5eb103a 100644
--- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
+++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
@@ -1310,6 +1310,21 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
checkAnswer(spark.sql("select * from t_carbn01b_hive"), spark.sql("select * from t_carbn01b"))
spark.sql("drop table if exists t_carbn01b_hive")
spark.sql(s"drop table if exists t_carbn01b")
+ }
+
+ test("Test Float value by having negative exponents") {
+ spark.sql("DROP TABLE IF EXISTS float_p")
+ spark.sql("DROP TABLE IF EXISTS float_c")
+ spark.sql("CREATE TABLE float_p(f float) using parquet")
+ spark.sql("CREATE TABLE float_c(f float) using carbon")
+ spark.sql("INSERT INTO float_p select \"1.4E-3\"")
+ spark.sql("INSERT INTO float_p select \"1.4E-38\"")
+ spark.sql("INSERT INTO float_c select \"1.4E-3\"")
+ spark.sql("INSERT INTO float_c select \"1.4E-38\"")
+ checkAnswer(spark.sql("SELECT * FROM float_p"),
+ spark.sql("SELECT * FROM float_c"))
+ spark.sql("DROP TABLE float_p")
+ spark.sql("DROP TABLE float_c")
}
test("test fileformat flow with drop and query on same table") {