You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/06/24 03:14:11 UTC
[carbondata] branch master updated: [CARBONDATA-3444]Fix MV query
failure when projection has cast expression with alias
This is an automated email from the ASF dual-hosted git repository.
kumarvishal09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 0d32c6b [CARBONDATA-3444]Fix MV query failure when projection has cast expression with alias
0d32c6b is described below
commit 0d32c6b8303f15e433fa3494e106f7ec6fa03b33
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Wed Jun 19 13:38:20 2019 +0530
[CARBONDATA-3444]Fix MV query failure when projection has cast expression with alias
Problem:
MV datamap creation fails when the project column as cast expression with multiple arithmetic functions on
one of main table columns with alias. It throws the field does not exists error.
when create datamap DDL has DM provider name as capital letters, the query was not hitting the MV table
Solution:
When making fieldRelationMap, handling the above case was missed, added a case to handle this scenario.
When loading the datamapCatalogs, take care to convert to lower case
This closes #3298
---
.../carbondata/core/datamap/DataMapStoreManager.java | 4 ++--
.../scala/org/apache/carbondata/mv/datamap/MVUtil.scala | 17 +++++++++++++++++
.../apache/carbondata/mv/rewrite/MVCreateTestCase.scala | 16 ++++++++++++++++
.../carbondata/mv/plans/modular/ModularRelation.scala | 1 +
4 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 729c419..a6a2031 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -240,7 +240,7 @@ public final class DataMapStoreManager {
if (dataMapCatalog == null) {
dataMapCatalog = dataMapProvider.createDataMapCatalog();
if (dataMapCatalog != null) {
- dataMapCatalogs.put(name, dataMapCatalog);
+ dataMapCatalogs.put(name.toLowerCase(), dataMapCatalog);
dataMapCatalog.registerSchema(dataMapSchema);
}
} else {
@@ -291,7 +291,7 @@ public final class DataMapStoreManager {
if (null == dataMapCatalog) {
throw new RuntimeException("Internal Error.");
}
- dataMapCatalogs.put(schema.getProviderName(), dataMapCatalog);
+ dataMapCatalogs.put(schema.getProviderName().toLowerCase(), dataMapCatalog);
}
try {
dataMapCatalog.registerSchema(schema);
diff --git a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVUtil.scala b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVUtil.scala
index 048e22d..4e633a6 100644
--- a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVUtil.scala
+++ b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVUtil.scala
@@ -113,6 +113,23 @@ class MVUtil {
}
case a@Alias(_, name) =>
checkIfComplexDataTypeExists(a)
+ val arrayBuffer: ArrayBuffer[ColumnTableRelation] = new ArrayBuffer[ColumnTableRelation]()
+ a.collect {
+ case attr: AttributeReference =>
+ val carbonTable = getCarbonTable(logicalRelation, attr)
+ if (null != carbonTable) {
+ val relation = getColumnRelation(attr.name,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableId,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableName,
+ carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getDatabaseName,
+ carbonTable)
+ if (null != relation) {
+ arrayBuffer += relation
+ }
+ }
+ }
+ fieldToDataMapFieldMap +=
+ getFieldToDataMapFields(a.name, a.dataType, None, "arithmetic", arrayBuffer, "")
}
fieldToDataMapFieldMap
}
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
index 535ddef..1d259c8 100644
--- a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/MVCreateTestCase.scala
@@ -1153,6 +1153,22 @@ class MVCreateTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table IF EXISTS maintable")
}
+ test("test cast expression with mv") {
+ sql("drop table IF EXISTS maintable")
+ sql("create table maintable (m_month bigint, c_code string, " +
+ "c_country smallint, d_dollar_value double, q_quantity double, u_unit smallint, b_country smallint, i_id int, y_year smallint) stored by 'carbondata'")
+ sql("insert into maintable select 10, 'xxx', 123, 456, 45, 5, 23, 1, 2000")
+ sql("drop datamap if exists da_cast")
+ sql(
+ "create datamap da_cast using 'mv' as select cast(floor((m_month +1000) / 900) * 900 - 2000 AS INT) as a, c_code as abc,m_month from maintable")
+ val df1 = sql(
+ " select cast(floor((m_month +1000) / 900) * 900 - 2000 AS INT) as a ,c_code as abc from maintable")
+ val df2 = sql(
+ " select cast(floor((m_month +1000) / 900) * 900 - 2000 AS INT),c_code as abc from maintable")
+ val analyzed1 = df1.queryExecution.analyzed
+ assert(TestUtil.verifyMVDataMap(analyzed1, "da_cast"))
+ }
+
def drop(): Unit = {
sql("drop table IF EXISTS fact_table1")
sql("drop table IF EXISTS fact_table2")
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
index b7512d2..32778d6 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
@@ -101,6 +101,7 @@ object HarmonizedRelation {
case alias: Alias =>
alias.child.isInstanceOf[AttributeReference] ||
alias.child.isInstanceOf[Literal] ||
+ alias.child.isInstanceOf[Expression] ||
(alias.child match {
case AggregateExpression(First(_, _), _, _, _) => true
case AggregateExpression(Last(_, _), _, _, _) => true