You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/01/31 05:22:36 UTC
[16/50] [abbrv] carbondata git commit: [CARBONDATA-2070]fix create
preaggregate on decimal column in hive metastore
[CARBONDATA-2070]fix create preaggregate on decimal column in hive metastore
Problem: when hive metastore is enabled and aggregate table is tried to create on the decimal column of main table, cast exception is thrown for Decimal datatype
solution:During creation of TableInfo from hivemetastore the DataMapSchemas and the columns
dataTypes are not converted to the appropriate child classes.
convert to actual datatype
This closes #1852
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee11bb1e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee11bb1e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee11bb1e
Branch: refs/heads/carbonstore
Commit: ee11bb1eb7790687b685b4af35ec9f1ab7fa6eff
Parents: 44ffaf5
Author: akashrn5 <ak...@gmail.com>
Authored: Tue Jan 23 18:59:51 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Jan 24 23:51:30 2018 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 11 +++++++++++
.../spark/testsuite/datamap/TestDataMapCommand.scala | 12 ++++++++++++
2 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dd65b84..4bb0d20 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -156,6 +156,17 @@ public class CarbonTable implements Serializable {
columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
columnSchema.getPrecision(), columnSchema.getScale()));
}
+ List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList();
+ for (DataMapSchema dataMapSchema : childSchema) {
+ if (dataMapSchema.childSchema != null
+ && dataMapSchema.childSchema.getListOfColumns().size() > 0) {
+ for (ColumnSchema columnSchema : dataMapSchema.childSchema.getListOfColumns()) {
+ columnSchema.setDataType(DataTypeUtil
+ .valueOf(columnSchema.getDataType(), columnSchema.getPrecision(),
+ columnSchema.getScale()));
+ }
+ }
+ }
if (tableInfo.getFactTable().getBucketingInfo() != null) {
for (ColumnSchema columnSchema : tableInfo.getFactTable()
.getBucketingInfo().getListOfColumns()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index f3458e2..a0ea317 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -35,6 +35,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists datamaptest")
sql("drop table if exists datamapshowtest")
+ sql("drop table if exists uniqdata")
sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
}
@@ -210,6 +211,16 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
}
+ test("test preaggregate load for decimal column for hivemetastore") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
+ sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"),Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ }
+
test("create pre-agg table with path") {
sql("drop table if exists main_preagg")
sql("drop table if exists main ")
@@ -237,6 +248,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
override def afterAll {
sql("DROP TABLE IF EXISTS maintable")
+ sql("drop table if exists uniqdata")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
sql("drop table if exists datamaptest")