You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/03/09 08:44:28 UTC
[10/12] carbondata git commit: [CARBONDATA-2242] Add Materialized
View modules
http://git-wip-us.apache.org/repos/asf/carbondata/blob/02fd7873/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
new file mode 100644
index 0000000..8e16b30
--- /dev/null
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/Tpcds_1_4_Suite.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.rewrite
+
+import org.apache.carbondata.mv.MQOSession
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.util._
+import org.scalatest.BeforeAndAfter
+import scala.util.{Failure, Success, Try}
+
+import org.apache.carbondata.mv.testutil.Tpcds_1_4_Tables._
+
+import org.apache.spark.sql.test.util.PlanTest
+
+class Tpcds_1_4_Suite extends PlanTest with BeforeAndAfter {
+
+ import org.apache.carbondata.mv.rewrite.matching.TestTPCDS_1_4_Batch._
+
+ val spark = SparkSession.builder().master("local").enableHiveSupport().getOrCreate()
+ val testHive = new org.apache.spark.sql.hive.test.TestHiveContext(spark.sparkContext, false)
+ val hiveClient = testHive.sparkSession.metadataHive
+
+ test("test using tpc-ds queries") {
+
+ tpcds1_4Tables.foreach { create_table =>
+ hiveClient.runSqlHive(create_table)
+ }
+
+// val dest = "case_30"
+// val dest = "case_32"
+ val dest = "case_3"
+
+ tpcds_1_4_testCases.foreach { testcase =>
+ if (testcase._1 == dest) {
+ val mqoSession = new MQOSession(testHive.sparkSession)
+ val summaryDF = testHive.sparkSession.sql(testcase._2)
+ mqoSession.sharedState.registerSummaryDataset(summaryDF)
+
+ Try(mqoSession.rewrite(testcase._3).withSummaryData) match {
+ case Success(rewrittenPlan) =>
+ println(s"""\n\n===== REWRITTEN MODULAR PLAN for ${testcase._1} =====\n\n$rewrittenPlan \n""")
+
+ Try(rewrittenPlan.asCompactSQL) match {
+ case Success(s) =>
+ println(s"\n\n===== CONVERTED SQL for ${testcase._1} =====\n\n${s}\n")
+ if (!s.trim.equals(testcase._4)) {
+ println(
+ s"""
+ |=== FAIL: SQLs do not match ===
+ |${sideBySide(s, testcase._4).mkString("\n")}
+ """.stripMargin)
+ }
+
+ case Failure(e) => println(s"""\n\n===== CONVERTED SQL for ${testcase._1} failed =====\n\n${e.toString}""")
+ }
+
+ case Failure(e) => println(s"""\n\n==== MODULARIZE the logical query plan for ${testcase._1} failed =====\n\n${e.toString}""")
+ }
+
+// val rewrittenSQL = rewrittenPlan.asCompactSQL
+// val rewrittenSQL = mqoSession.rewrite(testcase._3).toCompactSQL
+
+// if (!rewrittenSQL.equals(testcase._4)) {
+// fail(
+// s"""
+// |=== FAIL: SQLs do not match ===
+// |${sideBySide(rewrittenSQL, testcase._4).mkString("\n")}
+// """.stripMargin)
+// }
+ }
+
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/02fd7873/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
new file mode 100644
index 0000000..02bbff3
--- /dev/null
+++ b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestSQLBatch.scala
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.carbondata.mv.rewrite.matching
+
+object TestSQLBatch {
+
+ // seq of (summaryDataset(MV), testUserSQL(Q), correctRewrittenSQL)
+ val sampleTestCases = Seq(
+ ("case_1",
+ s"""
+ |SELECT i_item_id
+ |FROM Item
+ |WHERE i_item_sk = 1
+ """.stripMargin.trim,
+ s"""
+ |SELECT i_item_id, i_item_sk
+ |FROM Item
+ |WHERE i_item_sk = 1 and i_item_id > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.`i_item_id`, item.`i_item_sk`
+ |FROM
+ | item
+ |WHERE
+ | (item.`i_item_sk` = 1) AND (item.`i_item_id` > 0)
+ """.stripMargin.trim),
+ ("case_2",
+ s"""
+ |SELECT i_item_id
+ |FROM Item
+ |WHERE i_item_sk = 1
+ """.stripMargin.trim,
+ s"""
+ |SELECT i_item_id, i_item_sk
+ |FROM Item
+ |WHERE i_item_sk = 1 or i_item_id > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.`i_item_id`, item.`i_item_sk`
+ |FROM
+ | item
+ |WHERE
+ | ((item.`i_item_sk` = 1) OR (item.`i_item_id` > 0))
+ """.stripMargin.trim),
+ ("case_3",
+ s"""
+ |SELECT faid, flid, date
+ |FROM Fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, flid, year(date) as year
+ |FROM Fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`, gen_subsumer_0.`flid`, year(CAST(gen_subsumer_0.`date` AS DATE)) AS `year`
+ |FROM
+ | (SELECT fact.`faid`, fact.`flid`, fact.`date`
+ | FROM
+ | fact) gen_subsumer_0
+ """.stripMargin.trim),
+ ("case_4",
+ s"""
+ |SELECT faid, flid, date
+ |FROM Fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, flid
+ |FROM Fact
+ |WHERE year(date) = 2000
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`, gen_subsumer_0.`flid`
+ |FROM
+ | (SELECT fact.`faid`, fact.`flid`, fact.`date`
+ | FROM
+ | fact) gen_subsumer_0
+ |WHERE
+ | (year(CAST(gen_subsumer_0.`date` AS DATE)) = 2000)
+ """.stripMargin.trim),
+ ("case_5",
+ s"""
+ |SELECT faid, flid, date
+ |FROM Fact
+ |WHERE year(date) = 2000
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, flid
+ |FROM Fact
+ |WHERE year(date) = 2000
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`, gen_subsumer_0.`flid`
+ |FROM
+ | (SELECT fact.`faid`, fact.`flid`, fact.`date`
+ | FROM
+ | fact
+ | WHERE
+ | (year(CAST(fact.`date` AS DATE)) = 2000)) gen_subsumer_0
+ |WHERE
+ | (year(CAST(gen_subsumer_0.`date` AS DATE)) = 2000)
+ """.stripMargin.trim),
+ ("case_6",
+ s"""
+ |SELECT faid, flid, date
+ |FROM Fact
+ |WHERE year(date) in (2000,2001)
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, flid
+ |FROM Fact
+ |WHERE year(date) = 2000
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.`faid`, fact.`flid`
+ |FROM
+ | fact
+ |WHERE
+ | (year(CAST(fact.`date` AS DATE)) = 2000)
+ """.stripMargin.trim),
+ ("case_7",
+ s"""
+ |SELECT faid, flid, year(date) as year, count(*) as cnt
+ |FROM Fact
+ |GROUP BY faid, flid, year(date)
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid, year(date) as year, count(*) as cnt
+ |FROM Fact
+ |GROUP BY Fact.faid,year(Fact.date)
+ |HAVING count(*) > 2
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`, gen_subsumer_0.`year` AS `year`, sum(gen_subsumer_0.`cnt`) AS `cnt`
+ |FROM
+ | (SELECT fact.`faid`, fact.`flid`, year(CAST(fact.`date` AS DATE)) AS `year`, count(1) AS `cnt`
+ | FROM
+ | fact
+ | GROUP BY fact.`faid`, fact.`flid`, year(CAST(fact.`date` AS DATE))) gen_subsumer_0
+ |GROUP BY gen_subsumer_0.`faid`, gen_subsumer_0.`year`
+ |HAVING (sum(gen_subsumer_0.`cnt`) > 2L)
+ """.stripMargin.trim),
+ ("case_8",
+ s"""
+ |SELECT date
+ |FROM Fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT year(date)
+ |FROM Fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT year(CAST(gen_subsumer_0.`date` AS DATE)) AS `year(CAST(date AS DATE))`
+ |FROM
+ | (SELECT fact.`date`
+ | FROM
+ | fact) gen_subsumer_0
+ """.stripMargin.trim),
+ ("case_9",
+ s"""
+ |SELECT faid, flid
+ |FROM Fact
+ |WHERE faid > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid
+ |FROM Fact
+ |WHERE faid > 0 AND flid > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT gen_subsumer_0.`faid`
+ |FROM
+ | (SELECT fact.`faid`, fact.`flid`
+ | FROM
+ | fact
+ | WHERE
+ | (fact.`faid` > 0)) gen_subsumer_0
+ |WHERE
+ | (gen_subsumer_0.`faid` > 0) AND (gen_subsumer_0.`flid` > 0)
+ """.stripMargin.trim),
+ ("case_10",
+ s"""
+ |SELECT faid, flid
+ |FROM Fact
+ |WHERE faid > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT faid
+ |FROM Fact
+ |WHERE faid > 0 OR flid > 0
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.`faid`
+ |FROM
+ | fact
+ |WHERE
+ | ((fact.`faid` > 0) OR (fact.`flid` > 0))
+ """.stripMargin.trim))
+}
\ No newline at end of file