You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/20 07:29:22 UTC
[16/56] [abbrv] carbondata git commit: Convert decimal to byte at the
end of sort step when using GLOBAL_SORT.
Convert decimal to byte at the end of sort step when using GLOBAL_SORT.
Convert decimal to byte at the end of sort step when using GLOBAL_SORT.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b24f1228
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b24f1228
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b24f1228
Branch: refs/heads/streaming_ingest
Commit: b24f1228f8694e55792e2e90072e80c08e1cef7b
Parents: 4fb38bf
Author: Yadong Qi <qi...@gmail.com>
Authored: Wed Jun 14 14:22:23 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Wed Jun 14 20:01:16 2017 +0800
----------------------------------------------------------------------
.../dataload/TestGlobalSortDataLoad.scala | 56 ++++++++++++++++++++
.../newflow/sort/SortStepRowUtil.java | 19 ++++++-
2 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b24f1228/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index c00aaa3..190b7df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.test.TestQueryExecutor.projectPath
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
@@ -62,8 +63,10 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql("DROP TABLE IF EXISTS carbon_localsort_triple")
sql("DROP TABLE IF EXISTS carbon_localsort_delete")
sql("DROP TABLE IF EXISTS carbon_localsort_update")
+ sql("DROP TABLE IF EXISTS carbon_localsort_difftypes")
sql("DROP TABLE IF EXISTS carbon_globalsort")
sql("DROP TABLE IF EXISTS carbon_globalsort_partitioned")
+ sql("DROP TABLE IF EXISTS carbon_globalsort_difftypes")
}
// ----------------------------------- Compare Result -----------------------------------
@@ -274,6 +277,59 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql("SELECT * FROM carbon_localsort_once ORDER BY name"))
}
+ test("Test with different date types") {
+ val path = s"$projectPath/examples/spark2/src/main/resources/data.csv"
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_localsort_difftypes(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path' INTO TABLE carbon_localsort_difftypes
+ | OPTIONS('SORT_SCOPE'='GLOBAL_SORT',
+ | 'FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE TABLE carbon_globalsort_difftypes(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT
+ | )
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$path' INTO TABLE carbon_globalsort_difftypes
+ | OPTIONS('SORT_SCOPE'='GLOBAL_SORT',
+ | 'FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
+ """.stripMargin)
+
+ checkAnswer(sql("SELECT * FROM carbon_globalsort_difftypes ORDER BY shortField"),
+ sql("SELECT * FROM carbon_localsort_difftypes ORDER BY shortField"))
+ }
+
private def resetConf() {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.LOAD_SORT_SCOPE, CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b24f1228/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 474df4d..53f1497 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -17,6 +17,10 @@
package org.apache.carbondata.processing.newflow.sort;
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
import org.apache.carbondata.processing.util.NonDictionaryUtil;
@@ -54,9 +58,22 @@ public class SortStepRowUtil {
}
index = 0;
+
+ DataType[] measureDataType = parameters.getMeasureDataType();
// read measure values
for (int i = 0; i < measureCount; i++) {
- measures[index++] = data[allCount];
+ Object value = data[allCount];
+ if (null != value) {
+ if (measureDataType[i] == DataType.DECIMAL) {
+ BigDecimal decimal = (BigDecimal) value;
+ measures[index++] = DataTypeUtil.bigDecimalToByte(decimal);
+ } else {
+ measures[index++] = value;
+ }
+ } else {
+ measures[index++] = null;
+ }
+
allCount++;
}