You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2020/07/10 10:36:11 UTC
[carbondata] branch master updated: [CARBONDATA-3890] Fix MV case
sensitive issues with ImplicitCastInputTypes and add DOC for show MV
This is an automated email from the ASF dual-hosted git repository.
qiangcai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 65d9813 [CARBONDATA-3890] Fix MV case sensitive issues with ImplicitCastInputTypes and add DOC for show MV
65d9813 is described below
commit 65d9813652d76bd6b0c37e360618459224a08f21
Author: Indhumathi27 <in...@gmail.com>
AuthorDate: Fri Jul 3 21:09:49 2020 +0530
[CARBONDATA-3890] Fix MV case sensitive issues with ImplicitCastInputTypes and add DOC for show MV
Why is this PR needed?
Issue 1:
Queries having implicitCastInputTypes expressions is not hitting mv when expression is provided in Upper case
Issue 2:
MV document does not have info for SHOW MATERIALIZED VIEWS command
What changes were proposed in this PR?
Solution 1:
Transform castOrImplicitCastExp expression to change it's child attribute reference name to lower case while matching subsumer and subsume
Solution 2:
Added document for SHOW MATERIALIZED VIEWS command
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3823
---
docs/mv-guide.md | 21 ++++++++++++++++++++
.../org/apache/spark/sql/optimizer/MVMatcher.scala | 17 +++++++++++-----
.../view/rewrite/TestAllOperationsOnMV.scala | 23 ++++++++++++++++++++++
3 files changed, 56 insertions(+), 5 deletions(-)
diff --git a/docs/mv-guide.md b/docs/mv-guide.md
index 24e38b1..f6478a9 100644
--- a/docs/mv-guide.md
+++ b/docs/mv-guide.md
@@ -23,6 +23,7 @@
* [Querying Data](#querying-data)
* [Compaction](#compacting)
* [Data Management](#data-management)
+* [Show Materialized Views](#show-materialized-views)
* [Time Series Support](#time-series-support)
* [Time Series RollUp Support](#time-series-rollup-support)
@@ -221,6 +222,26 @@
Basically, user can manually trigger the operation by re-building the materialized view.
+### Show Materialized Views
+
+ Command syntax:
+ ```
+ SHOW MATERIALIZED VIEWS [ON TABLE [db_name.]table_name]
+ ```
+
+SHOW MATERIALIZED VIEWS command will display the information about all the materialized
+views created on the database or on the carbon table.
+The current information includes:
+
+ | Column Info | Description |
+ |-----------------------------|----------------------------------------------------------------------------|
+ | Database | Materialized view database name |
+ | Name | Materialized view name |
+ | Status | ENABLED / DISABLED |
+ | Refresh Mode | FULL / INCREMENTAL refresh to MV |
+ | Refresh Trigger Mode | ON_COMMIT / ON_MANUAL refresh to MV provided by user |
+ | Properties | Table properties of the materialized view |
+
## Time Series Support
Time series data are simply measurements or events that are tracked, monitored, down sampled, and
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
index d862920..4aee148 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
@@ -93,7 +93,10 @@ private abstract class MVMatchPattern extends Logging {
case function: ScalaUDF if function.function.isInstanceOf[TimeSeriesFunction] =>
getTransformedTimeSeriesFunction(function) -> alias.toAttribute
case cast: Cast if cast.child.isInstanceOf[AttributeReference] =>
- getTransformedCastExpression(cast) -> alias.toAttribute
+ getTransformedCastOrImplicitCastExpression(cast) -> alias.toAttribute
+ case implicitCastInputTypeExp: ImplicitCastInputTypes =>
+ getTransformedCastOrImplicitCastExpression(implicitCastInputTypeExp) ->
+ alias.toAttribute
case _ =>
alias.child -> alias.toAttribute
}
@@ -120,7 +123,9 @@ private abstract class MVMatchPattern extends Logging {
case function: ScalaUDF if function.function.isInstanceOf[TimeSeriesFunction] =>
getTransformedTimeSeriesFunction(function)
case cast: Cast if cast.child.isInstanceOf[AttributeReference] =>
- getTransformedCastExpression(cast)
+ getTransformedCastOrImplicitCastExpression(cast)
+ case implicitCastInputTypeExp: ImplicitCastInputTypes =>
+ getTransformedCastOrImplicitCastExpression(implicitCastInputTypeExp)
}
attribute = aliasMapExp.get(newExp)
}
@@ -247,10 +252,12 @@ private abstract class MVMatchPattern extends Logging {
}
/**
- * transform cast expression to change it's child attribute reference name to lower case
+ * transform castOrImplicitCastExp expression to change it's child attribute reference name to
+ * lower case
*/
- protected def getTransformedCastExpression(cast: Cast): Expression = {
- cast.transform {
+ protected def getTransformedCastOrImplicitCastExpression(
+ castOrImplicitCastExp: Expression): Expression = {
+ castOrImplicitCastExp.transform {
case reference: AttributeReference =>
CarbonToSparkAdapter.createAttributeReference(
reference.name.toLowerCase,
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
index c1d7d0e..bb6fe56 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/view/rewrite/TestAllOperationsOnMV.scala
@@ -678,6 +678,29 @@ class TestAllOperationsOnMV extends QueryTest with BeforeAndAfterEach {
sql("drop table IF EXISTS maintable")
}
+ test("test case sensitive issues with implicit cast type expressions") {
+ sql("drop table IF EXISTS maintable")
+ sql("CREATE TABLE maintable (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
+ "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 int) STORED AS carbondata")
+ sql("insert into maintable select 1,'abc','2001','2017-09-01 00:00:00','2017-09-03 00:00:00',1234567,1234564,'1234.456','1234.4567',1.123455,1.123455,null")
+ sql("drop materialized view if exists mv1")
+ sql("create materialized view mv1 as select length(CUST_NAME) from maintable where CUST_ID IS NULL or DOB IS NOT NULL or BIGINT_COLUMN1=1234567 or" +
+ " DECIMAL_COLUMN1=1234.456 or Double_COLUMN1=1.123455 or INTEGER_COLUMN1 IS NULL")
+ val withUpperCase = sql(
+ "select length(CUST_NAME) from maintable where CUST_ID IS NULL or DOB IS NOT NULL or " +
+ "BIGINT_COLUMN1=1234567 or DECIMAL_COLUMN1=1234.456 or Double_COLUMN1=1.123455 or INTEGER_COLUMN1 IS NULL")
+ val withLowerCase = sql(
+ "select length(cust_name) from maintable where cust_id IS NULL or dob IS NOT NULL or " +
+ "bigint_column1=1234567 or decimal_column1=1234.456 or double_column1=1.123455 or integer_column1 IS NULL")
+ checkAnswer(withUpperCase, Seq(Row(3)))
+ checkAnswer(withLowerCase, Seq(Row(3)))
+ TestUtil.verifyMVHit(withUpperCase.queryExecution.optimizedPlan, "mv1")
+ TestUtil.verifyMVHit(withLowerCase.queryExecution.optimizedPlan, "mv1")
+ sql("drop table IF EXISTS maintable")
+ }
+
test("drop meta cache on mv materialized view table") {
defaultConfig()
sql("drop table IF EXISTS maintable")