You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/07/20 10:13:46 UTC
[18/50] [abbrv] incubator-carbondata git commit: fix query avg of
bigint field (#807)
fix query avg of bigint field (#807)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6d2456f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6d2456f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6d2456f1
Branch: refs/heads/master
Commit: 6d2456f14479a324f9d5124acd20b739f1f0bb23
Parents: eecbdfd
Author: Gin-zhj <zh...@huawei.com>
Authored: Sat Jul 16 01:37:20 2016 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Fri Jul 15 23:07:20 2016 +0530
----------------------------------------------------------------------
.../aggregator/impl/AvgLongAggregator.java | 20 ++++---
.../carbondata/spark/agg/CarbonAggregates.scala | 2 +-
.../spark/src/test/resources/bigIntData.csv | 14 +++++
.../testsuite/bigdecimal/TestAvgForBigInt.scala | 59 ++++++++++++++++++++
4 files changed, 87 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6d2456f1/core/src/main/java/org/carbondata/query/aggregator/impl/AvgLongAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/AvgLongAggregator.java b/core/src/main/java/org/carbondata/query/aggregator/impl/AvgLongAggregator.java
index d608c90..b8a0179 100644
--- a/core/src/main/java/org/carbondata/query/aggregator/impl/AvgLongAggregator.java
+++ b/core/src/main/java/org/carbondata/query/aggregator/impl/AvgLongAggregator.java
@@ -89,10 +89,16 @@ public class AvgLongAggregator extends AbstractMeasureAggregatorBasic {
return buffer.array();
}
- @Override public Long getLongValue() {
- return aggVal / (long) count;
- }
+ /**
+ * Return the average of the aggregate values
+ *
+ * @return average aggregate value
+ */
+ // using double type when calculate avg, to avoid lost precision
+ @Override public Double getDoubleValue() {
+ return aggVal / count;
+ }
/**
* This method merge the aggregated value, in average aggregator it will add
* count and aggregate value
@@ -147,8 +153,8 @@ public class AvgLongAggregator extends AbstractMeasureAggregatorBasic {
}
@Override public int compareTo(MeasureAggregator o) {
- long val = getLongValue();
- long otherVal = o.getLongValue();
+ double val = getDoubleValue();
+ double otherVal = o.getDoubleValue();
if (val > otherVal) {
return 1;
} else if (val < otherVal) {
@@ -163,11 +169,11 @@ public class AvgLongAggregator extends AbstractMeasureAggregatorBasic {
return false;
}
AvgLongAggregator o = (AvgLongAggregator)obj;
- return getLongValue().equals(o.getLongValue());
+ return getDoubleValue().equals(o.getDoubleValue());
}
@Override public int hashCode() {
- return getLongValue().hashCode();
+ return getDoubleValue().hashCode();
}
@Override public void merge(byte[] value) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6d2456f1/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
index 61cf350..1a52688 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
@@ -368,7 +368,7 @@ case class AverageFunctionCarbon(expr: Expression, base: AggregateExpression1, f
case avg: AvgBigDecimalAggregator =>
Cast(Literal(avg.getBigDecimalValue), base.dataType).eval(null)
case avg: AvgLongAggregator =>
- Cast(Literal(avg.getLongValue), base.dataType).eval(null)
+ Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
case avg: AvgTimestampAggregator =>
Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
case _ =>
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6d2456f1/integration/spark/src/test/resources/bigIntData.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/bigIntData.csv b/integration/spark/src/test/resources/bigIntData.csv
new file mode 100644
index 0000000..443a514
--- /dev/null
+++ b/integration/spark/src/test/resources/bigIntData.csv
@@ -0,0 +1,14 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,10000001
+2,2015/7/24,china,aaa2,phone756,ASD42892,10000000
+3,2015/7/25,china,aaa3,phone1904,ASD37014,10000000
+4,2015/7/26,china,aaa4,phone2435,ASD66902,10000000
+5,2015/7/27,china,aaa5,phone2441,ASD90633,10000000
+6,2015/7/28,china,aaa6,phone294,ASD59961,10000000
+7,2015/7/29,china,aaa7,phone610,ASD14875,10000000
+8,2015/7/30,china,aaa8,phone1848,ASD57308,10000000
+9,2015/7/18,china,aaa9,phone706,ASD86717,10000000
+10,2015/7/19,usa,aaa10,phone685,ASD30505,10000000
+11,2015/7/18,china,aaa11,phone1554,ASD26101,10000000
+12,2015/7/19,china,aaa12,phone1781,ASD85711,10000000
+13,2015/7/19,china,aaa13,phone17851,ASD85721,10000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6d2456f1/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestAvgForBigInt.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestAvgForBigInt.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestAvgForBigInt.scala
new file mode 100644
index 0000000..513e13d
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestAvgForBigInt.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.spark.testsuite.bigdecimal
+
+import java.io.File
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class TestAvgForBigInt extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("drop table if exists carbonTable")
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ val csvFilePath = currentDirectory + "/src/test/resources/bigIntData.csv"
+
+ sql(
+ """
+ CREATE TABLE IF NOT EXISTS carbonTable (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary bigint)
+ STORED BY 'org.apache.carbondata.format'
+ """
+ )
+
+ sql(
+ "LOAD DATA LOCAL INPATH '" + csvFilePath + "' into table carbonTable"
+ )
+ }
+
+ test("test avg function on big int column") {
+ checkAnswer(
+ sql("select avg(salary) from carbonTable"),
+ sql("select sum(salary)/count(salary) from carbonTable")
+ )
+ }
+
+ override def afterAll {
+ sql("drop table if exists carbonTable")
+ }
+}