You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:27 UTC
[29/47] incubator-carbondata git commit: [CARBONDATA-113][Bug]fix
precision of DecimalType for CarbonAggregates(#875)
[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)
[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4d2f6842
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4d2f6842
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4d2f6842
Branch: refs/heads/master
Commit: 4d2f68427c6b3b2fc95175ebab16952e0999b939
Parents: adba597
Author: Gin-zhj <zh...@huawei.com>
Authored: Thu Jul 28 19:51:28 2016 +0800
Committer: david <qi...@qq.com>
Committed: Thu Jul 28 19:51:28 2016 +0800
----------------------------------------------------------------------
.../carbondata/spark/agg/CarbonAggregates.scala | 28 +++++++-------------
.../carbondata/spark/util/CarbonScalaUtil.scala | 16 +++++------
.../testsuite/bigdecimal/TestBigDecimal.scala | 23 +++++++++++++++-
3 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
index de685ab..50872ff 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
@@ -116,7 +116,7 @@ case class AverageCarbon(child: Expression, castedDataType: DataType = null)
AverageCarbonFinal(partialSum.toAttribute,
child.dataType match {
case IntegerType | StringType | LongType | TimestampType => DoubleType
- case _ => child.dataType
+ case _ => CarbonScalaUtil.updateDataType(child.dataType)
}),
partialSum :: Nil)
}
@@ -153,7 +153,11 @@ case class SumCarbon(child: Expression, castedDataType: DataType = null)
val partialSum = Alias(SumCarbon(child), "PartialSum")()
SplitEvaluation(
SumCarbonFinal(partialSum.toAttribute,
- if (castedDataType != null) castedDataType else child.dataType),
+ if (castedDataType != null) {
+ castedDataType
+ } else {
+ CarbonScalaUtil.updateDataType(child.dataType)
+ }),
partialSum :: Nil)
}
@@ -265,7 +269,7 @@ case class SumDistinctCarbon(child: Expression, castedDataType: DataType = null)
if (castedDataType != null) {
castedDataType
} else {
- child.dataType
+ CarbonScalaUtil.updateDataType(child.dataType)
}),
partialSum :: Nil)
}
@@ -369,10 +373,7 @@ case class AverageFunctionCarbon(expr: Expression, base: AggregateExpression1, f
} else {
avg match {
case avg: AvgBigDecimalAggregator =>
- val decimalValue: BigDecimal = avg.getBigDecimalValue
- val updatedDataType = CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
- Cast(Literal(decimalValue), updatedDataType).eval(null)
+ Cast(Literal(avg.getBigDecimalValue), base.dataType).eval(null)
case avg: AvgLongAggregator =>
Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
case avg: AvgTimestampAggregator =>
@@ -489,10 +490,7 @@ case class SumFunctionCarbon(expr: Expression, base: AggregateExpression1, final
} else {
sum match {
case s: SumBigDecimalAggregator =>
- val decimalValue: BigDecimal = sum.getBigDecimalValue
- val updatedDataType = CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
- Cast(Literal(decimalValue), updatedDataType).eval(input)
+ Cast(Literal(sum.getBigDecimalValue), base.dataType).eval(input)
case s: SumLongAggregator =>
Cast(Literal(sum.getLongValue), base.dataType).eval(input)
case _ =>
@@ -689,13 +687,7 @@ case class SumDisctinctFunctionCarbon(expr: Expression, base: AggregateExpressio
null
}
else {
- val updatedDataType = base.dataType match {
- case decimal: DecimalType =>
- CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(distinct.getBigDecimalValue, base.dataType)
- case _ => base.dataType
- }
- Cast(Literal(distinct.getValueObject), updatedDataType).eval(null)
+ Cast(Literal(distinct.getValueObject), base.dataType).eval(null)
}
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 20d45dc..b3effd3 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -105,19 +105,17 @@ object CarbonScalaUtil {
}
}
- def getDecimalDataTypeWithUpdatedPrecision(decimalValue: java.math.BigDecimal,
+ def updateDataType(
currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = {
- var newDataType: org.apache.spark.sql.types.DataType = currentDataType
- if (null != decimalValue) {
- val precision = decimalValue.precision
- if (precision <= DecimalType.MAX_PRECISION) {
- newDataType = DecimalType(precision, decimalValue.scale())
- }
+ currentDataType match {
+ case decimal: DecimalType =>
+ val scale = currentDataType.asInstanceOf[DecimalType].scale
+ DecimalType(DecimalType.MAX_PRECISION, scale)
+ case _ =>
+ currentDataType
}
- newDataType
}
-
case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
object CarbonSparkUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index b07705d..fb28568 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -136,7 +136,28 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists decimalDictLookUp")
}
-
+
+ test("test sum aggregation on big decimal column with high precision") {
+ sql("drop table if exists carbonBigDecimal")
+ sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
+
+ checkAnswer(sql("select sum(salary)+10 from carbonBigDecimal"),
+ sql("select sum(salary)+10 from hiveBigDecimal"))
+
+ sql("drop table if exists carbonBigDecimal")
+ }
+
+ test("test sum-distinct aggregation on big decimal column with high precision") {
+ sql("drop table if exists carbonBigDecimal")
+ sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
+
+ checkAnswer(sql("select sum(distinct(salary))+10 from carbonBigDecimal"),
+ sql("select sum(distinct(salary))+10 from hiveBigDecimal"))
+
+ sql("drop table if exists carbonBigDecimal")
+ }
override def afterAll {
sql("drop table if exists carbonTable")