You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/05/12 15:01:59 UTC
[1/4] carbondata git commit: [CARBONDATA-2474] Support Modular Plan
for Materialized View DataMap
Repository: carbondata
Updated Branches:
refs/heads/master 443b717d2 -> ffddba704
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
new file mode 100644
index 0000000..97772c7
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
@@ -0,0 +1,819 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+object Tpcds_1_4_Tables {
+ val tpcds1_4Tables = Seq[String](
+ s"""
+ |CREATE TABLE catalog_sales (
+ | `cs_sold_date_sk` int,
+ | `cs_sold_time_sk` int,
+ | `cs_ship_date_sk` int,
+ | `cs_bill_customer_sk` int,
+ | `cs_bill_cdemo_sk` int,
+ | `cs_bill_hdemo_sk` int,
+ | `cs_bill_addr_sk` int,
+ | `cs_ship_customer_sk` int,
+ | `cs_ship_cdemo_sk` int,
+ | `cs_ship_hdemo_sk` int,
+ | `cs_ship_addr_sk` int,
+ | `cs_call_center_sk` int,
+ | `cs_catalog_page_sk` int,
+ | `cs_ship_mode_sk` int,
+ | `cs_warehouse_sk` int,
+ | `cs_item_sk` int,
+ | `cs_promo_sk` int,
+ | `cs_order_number` bigint,
+ | `cs_quantity` int,
+ | `cs_wholesale_cost` decimal(7,2),
+ | `cs_list_price` decimal(7,2),
+ | `cs_sales_price` decimal(7,2),
+ | `cs_ext_discount_amt` decimal(7,2),
+ | `cs_ext_sales_price` decimal(7,2),
+ | `cs_ext_wholesale_cost` decimal(7,2),
+ | `cs_ext_list_price` decimal(7,2),
+ | `cs_ext_tax` decimal(7,2),
+ | `cs_coupon_amt` decimal(7,2),
+ | `cs_ext_ship_cost` decimal(7,2),
+ | `cs_net_paid` decimal(7,2),
+ | `cs_net_paid_inc_tax` decimal(7,2),
+ | `cs_net_paid_inc_ship` decimal(7,2),
+ | `cs_net_paid_inc_ship_tax` decimal(7,2),
+ | `cs_net_profit` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE catalog_returns (
+ | `cr_returned_date_sk` int,
+ | `cr_returned_time_sk` int,
+ | `cr_item_sk` int,
+ | `cr_refunded_customer_sk` int,
+ | `cr_refunded_cdemo_sk` int,
+ | `cr_refunded_hdemo_sk` int,
+ | `cr_refunded_addr_sk` int,
+ | `cr_returning_customer_sk` int,
+ | `cr_returning_cdemo_sk` int,
+ | `cr_returning_hdemo_sk` int,
+ | `cr_returning_addr_sk` int,
+ | `cr_call_center_sk` int,
+ | `cr_catalog_page_sk` int,
+ | `cr_ship_mode_sk` int,
+ | `cr_warehouse_sk` int,
+ | `cr_reason_sk` int,
+ | `cr_order_number` bigint,
+ | `cr_return_quantity` int,
+ | `cr_return_amount` decimal(7,2),
+ | `cr_return_tax` decimal(7,2),
+ | `cr_return_amt_inc_tax` decimal(7,2),
+ | `cr_fee` decimal(7,2),
+ | `cr_return_ship_cost` decimal(7,2),
+ | `cr_refunded_cash` decimal(7,2),
+ | `cr_reversed_charge` decimal(7,2),
+ | `cr_store_credit` decimal(7,2),
+ | `cr_net_loss` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE inventory (
+ | `inv_date_sk` int,
+ | `inv_item_sk` int,
+ | `inv_warehouse_sk` int,
+ | `inv_quantity_on_hand` int
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE store_sales (
+ | `ss_sold_date_sk` int,
+ | `ss_sold_time_sk` int,
+ | `ss_item_sk` int,
+ | `ss_customer_sk` int,
+ | `ss_cdemo_sk` int,
+ | `ss_hdemo_sk` int,
+ | `ss_addr_sk` int,
+ | `ss_store_sk` int,
+ | `ss_promo_sk` int,
+ | `ss_ticket_number` bigint,
+ | `ss_quantity` int,
+ | `ss_wholesale_cost` decimal(7,2),
+ | `ss_list_price` decimal(7,2),
+ | `ss_sales_price` decimal(7,2),
+ | `ss_ext_discount_amt` decimal(7,2),
+ | `ss_ext_sales_price` decimal(7,2),
+ | `ss_ext_wholesale_cost` decimal(7,2),
+ | `ss_ext_list_price` decimal(7,2),
+ | `ss_ext_tax` decimal(7,2),
+ | `ss_coupon_amt` decimal(7,2),
+ | `ss_net_paid` decimal(7,2),
+ | `ss_net_paid_inc_tax` decimal(7,2),
+ | `ss_net_profit` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE store_returns (
+ | `sr_returned_date_sk` int,
+ | `sr_return_time_sk` int,
+ | `sr_item_sk` int,
+ | `sr_customer_sk` int,
+ | `sr_cdemo_sk` int,
+ | `sr_hdemo_sk` int,
+ | `sr_addr_sk` int,
+ | `sr_store_sk` int,
+ | `sr_reason_sk` int,
+ | `sr_ticket_number` bigint,
+ | `sr_return_quantity` int,
+ | `sr_return_amt` decimal(7,2),
+ | `sr_return_tax` decimal(7,2),
+ | `sr_return_amt_inc_tax` decimal(7,2),
+ | `sr_fee` decimal(7,2),
+ | `sr_return_ship_cost` decimal(7,2),
+ | `sr_refunded_cash` decimal(7,2),
+ | `sr_reversed_charge` decimal(7,2),
+ | `sr_store_credit` decimal(7,2),
+ | `sr_net_loss` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE web_sales (
+ | `ws_sold_date_sk` int,
+ | `ws_sold_time_sk` int,
+ | `ws_ship_date_sk` int,
+ | `ws_item_sk` int,
+ | `ws_bill_customer_sk` int,
+ | `ws_bill_cdemo_sk` int,
+ | `ws_bill_hdemo_sk` int,
+ | `ws_bill_addr_sk` int,
+ | `ws_ship_customer_sk` int,
+ | `ws_ship_cdemo_sk` int,
+ | `ws_ship_hdemo_sk` int,
+ | `ws_ship_addr_sk` int,
+ | `ws_web_page_sk` int,
+ | `ws_web_site_sk` int,
+ | `ws_ship_mode_sk` int,
+ | `ws_warehouse_sk` int,
+ | `ws_promo_sk` int,
+ | `ws_order_number` bigint,
+ | `ws_quantity` int,
+ | `ws_wholesale_cost` decimal(7,2),
+ | `ws_list_price` decimal(7,2),
+ | `ws_sales_price` decimal(7,2),
+ | `ws_ext_discount_amt` decimal(7,2),
+ | `ws_ext_sales_price` decimal(7,2),
+ | `ws_ext_wholesale_cost` decimal(7,2),
+ | `ws_ext_list_price` decimal(7,2),
+ | `ws_ext_tax` decimal(7,2),
+ | `ws_coupon_amt` decimal(7,2),
+ | `ws_ext_ship_cost` decimal(7,2),
+ | `ws_net_paid` decimal(7,2),
+ | `ws_net_paid_inc_tax` decimal(7,2),
+ | `ws_net_paid_inc_ship` decimal(7,2),
+ | `ws_net_paid_inc_ship_tax` decimal(7,2),
+ | `ws_net_profit` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE web_returns (
+ | `wr_returned_date_sk` int,
+ | `wr_returned_time_sk` int,
+ | `wr_item_sk` int,
+ | `wr_refunded_customer_sk` int,
+ | `wr_refunded_cdemo_sk` int,
+ | `wr_refunded_hdemo_sk` int,
+ | `wr_refunded_addr_sk` int,
+ | `wr_returning_customer_sk` int,
+ | `wr_returning_cdemo_sk` int,
+ | `wr_returning_hdemo_sk` int,
+ | `wr_returning_addr_sk` int,
+ | `wr_web_page_sk` int,
+ | `wr_reason_sk` int,
+ | `wr_order_number` bigint,
+ | `wr_return_quantity` int,
+ | `wr_return_amt` decimal(7,2),
+ | `wr_return_tax` decimal(7,2),
+ | `wr_return_amt_inc_tax` decimal(7,2),
+ | `wr_fee` decimal(7,2),
+ | `wr_return_ship_cost` decimal(7,2),
+ | `wr_refunded_cash` decimal(7,2),
+ | `wr_reversed_charge` decimal(7,2),
+ | `wr_account_credit` decimal(7,2),
+ | `wr_net_loss` decimal(7,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE call_center (
+ | `cc_call_center_sk` int,
+ | `cc_call_center_id` string,
+ | `cc_rec_start_date` date,
+ | `cc_rec_end_date` date,
+ | `cc_closed_date_sk` int,
+ | `cc_open_date_sk` int,
+ | `cc_name` string,
+ | `cc_class` string,
+ | `cc_employees` int,
+ | `cc_sq_ft` int,
+ | `cc_hours` string,
+ | `cc_manager` string,
+ | `cc_mkt_id` int,
+ | `cc_mkt_class` string,
+ | `cc_mkt_desc` string,
+ | `cc_market_manager` string,
+ | `cc_division` int,
+ | `cc_division_name` string,
+ | `cc_company` int,
+ | `cc_company_name` string,
+ | `cc_street_number` string,
+ | `cc_street_name` string,
+ | `cc_street_type` string,
+ | `cc_suite_number` string,
+ | `cc_city` string,
+ | `cc_county` string,
+ | `cc_state` string,
+ | `cc_zip` string,
+ | `cc_country` string,
+ | `cc_gmt_offset` decimal(5,2),
+ | `cc_tax_percentage` decimal(5,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE catalog_page (
+ | `cp_catalog_page_sk` int,
+ | `cp_catalog_page_id` string,
+ | `cp_start_date_sk` int,
+ | `cp_end_date_sk` int,
+ | `cp_department` string,
+ | `cp_catalog_number` int,
+ | `cp_catalog_page_number` int,
+ | `cp_description` string,
+ | `cp_type` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE customer (
+ | `c_customer_sk` int,
+ | `c_customer_id` string,
+ | `c_current_cdemo_sk` int,
+ | `c_current_hdemo_sk` int,
+ | `c_current_addr_sk` int,
+ | `c_first_shipto_date_sk` int,
+ | `c_first_sales_date_sk` int,
+ | `c_salutation` string,
+ | `c_first_name` string,
+ | `c_last_name` string,
+ | `c_preferred_cust_flag` string,
+ | `c_birth_day` int,
+ | `c_birth_month` int,
+ | `c_birth_year` int,
+ | `c_birth_country` string,
+ | `c_login` string,
+ | `c_email_address` string,
+ | `c_last_review_date` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE customer_address (
+ | `ca_address_sk` int,
+ | `ca_address_id` string,
+ | `ca_street_number` string,
+ | `ca_street_name` string,
+ | `ca_street_type` string,
+ | `ca_suite_number` string,
+ | `ca_city` string,
+ | `ca_county` string,
+ | `ca_state` string,
+ | `ca_zip` string,
+ | `ca_country` string,
+ | `ca_gmt_offset` decimal(5,2),
+ | `ca_location_type` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE customer_demographics (
+ | `cd_demo_sk` int,
+ | `cd_gender` string,
+ | `cd_marital_status` string,
+ | `cd_education_status` string,
+ | `cd_purchase_estimate` int,
+ | `cd_credit_rating` string,
+ | `cd_dep_count` int,
+ | `cd_dep_employed_count` int,
+ | `cd_dep_college_count` int
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE date_dim (
+ | `d_date_sk` int,
+ | `d_date_id` string,
+ | `d_date` date,
+ | `d_month_seq` int,
+ | `d_week_seq` int,
+ | `d_quarter_seq` int,
+ | `d_year` int,
+ | `d_dow` int,
+ | `d_moy` int,
+ | `d_dom` int,
+ | `d_qoy` int,
+ | `d_fy_year` int,
+ | `d_fy_quarter_seq` int,
+ | `d_fy_week_seq` int,
+ | `d_day_name` string,
+ | `d_quarter_name` string,
+ | `d_holiday` string,
+ | `d_weekend` string,
+ | `d_following_holiday` string,
+ | `d_first_dom` int,
+ | `d_last_dom` int,
+ | `d_same_day_ly` int,
+ | `d_same_day_lq` int,
+ | `d_current_day` string,
+ | `d_current_week` string,
+ | `d_current_month` string,
+ | `d_current_quarter` string,
+ | `d_current_year` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE household_demographics (
+ | `hd_demo_sk` int,
+ | `hd_income_band_sk` int,
+ | `hd_buy_potential` string,
+ | `hd_dep_count` int,
+ | `hd_vehicle_count` int
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE income_band (
+ | `ib_income_band_sk` int,
+ | `ib_lower_bound` int,
+ | `ib_upper_bound` int
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE item (
+ | `i_item_sk` int,
+ | `i_item_id` string,
+ | `i_rec_start_date` date,
+ | `i_rec_end_date` date,
+ | `i_item_desc` string,
+ | `i_current_price` decimal(7,2),
+ | `i_wholesale_cost` decimal(7,2),
+ | `i_brand_id` int,
+ | `i_brand` string,
+ | `i_class_id` int,
+ | `i_class` string,
+ | `i_category_id` int,
+ | `i_category` string,
+ | `i_manufact_id` int,
+ | `i_manufact` string,
+ | `i_size` string,
+ | `i_formulation` string,
+ | `i_color` string,
+ | `i_units` string,
+ | `i_container` string,
+ | `i_manager_id` int,
+ | `i_product_name` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE promotion (
+ | `p_promo_sk` int,
+ | `p_promo_id` string,
+ | `p_start_date_sk` int,
+ | `p_end_date_sk` int,
+ | `p_item_sk` int,
+ | `p_cost` decimal(15,2),
+ | `p_response_target` int,
+ | `p_promo_name` string,
+ | `p_channel_dmail` string,
+ | `p_channel_email` string,
+ | `p_channel_catalog` string,
+ | `p_channel_tv` string,
+ | `p_channel_radio` string,
+ | `p_channel_press` string,
+ | `p_channel_event` string,
+ | `p_channel_demo` string,
+ | `p_channel_details` string,
+ | `p_purpose` string,
+ | `p_discount_active` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE reason (
+ | `r_reason_sk` int,
+ | `r_reason_id` string,
+ | `r_reason_desc` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE ship_mode (
+ | `sm_ship_mode_sk` int,
+ | `sm_ship_mode_id` string,
+ | `sm_type` string,
+ | `sm_code` string,
+ | `sm_carrier` string,
+ | `sm_contract` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE store (
+ | `s_store_sk` int,
+ | `s_store_id` string,
+ | `s_rec_start_date` date,
+ | `s_rec_end_date` date,
+ | `s_closed_date_sk` int,
+ | `s_store_name` string,
+ | `s_number_employees` int,
+ | `s_floor_space` int,
+ | `s_hours` string,
+ | `s_manager` string,
+ | `s_market_id` int,
+ | `s_geography_class` string,
+ | `s_market_desc` string,
+ | `s_market_manager` string,
+ | `s_division_id` int,
+ | `s_division_name` string,
+ | `s_company_id` int,
+ | `s_company_name` string,
+ | `s_street_number` string,
+ | `s_street_name` string,
+ | `s_street_type` string,
+ | `s_suite_number` string,
+ | `s_city` string,
+ | `s_county` string,
+ | `s_state` string,
+ | `s_zip` string,
+ | `s_country ` string,
+ | `s_gmt_offset` decimal(5,2),
+ | `s_tax_precentage` decimal(5,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE time_dim (
+ | `t_time_sk` int,
+ | `t_time_id` string,
+ | `t_time` int,
+ | `t_hour` int,
+ | `t_minute` int,
+ | `t_second` int,
+ | `t_am_pm` string,
+ | `t_shift` string,
+ | `t_sub_shift` string,
+ | `t_meal_time` string
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE warehouse (
+ | `w_warehouse_sk` int,
+ | `w_warehouse_id` string,
+ | `w_warehouse_name` string,
+ | `w_warehouse_sq_ft` int,
+ | `w_street_number` string,
+ | `w_street_name` string,
+ | `w_street_type` string,
+ | `w_suite_number` string,
+ | `w_city` string,
+ | `w_county` string,
+ | `w_state` string,
+ | `w_zip` string,
+ | `w_country` string,
+ | `w_gmt_offset` decimal(5,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE web_page (
+ | `wp_web_page_sk` int,
+ | `wp_web_page_id` string,
+ | `wp_rec_start_date` date,
+ | `wp_rec_end_date` date,
+ | `wp_creation_date_sk` int,
+ | `wp_access_date_sk` int,
+ | `wp_autogen_flag` string,
+ | `wp_customer_sk` int,
+ | `wp_url` string,
+ | `wp_type` string,
+ | `wp_char_count` int,
+ | `wp_link_count` int,
+ | `wp_image_count` int,
+ | `wp_max_ad_count` int
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE web_site (
+ | `web_site_sk` int,
+ | `web_site_id` string,
+ | `web_rec_start_date` date,
+ | `web_rec_end_date` date,
+ | `web_name` string,
+ | `web_open_date_sk` int,
+ | `web_close_date_sk` int,
+ | `web_class` string,
+ | `web_manager` string,
+ | `web_mkt_id` int,
+ | `web_mkt_class` string,
+ | `web_mkt_desc` string,
+ | `web_market_manager` string,
+ | `web_company_id` int,
+ | `web_company_name` string,
+ | `web_street_number` string,
+ | `web_street_name` string,
+ | `web_street_type` string,
+ | `web_suite_number` string,
+ | `web_city` string,
+ | `web_county` string,
+ | `web_state` string,
+ | `web_zip` string,
+ | `web_country` string,
+ | `web_gmt_offset` decimal(5,2),
+ | `web_tax_percentage` decimal(5,2)
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE sdr_dyn_seq_custer_iot_all_hour_60min
+ |(
+ | `dim_1` String,
+ | `dim_51` String,
+ | `starttime` String,
+ | `dim_2` String,
+ | `dim_3` String,
+ | `dim_4` String,
+ | `dim_5` String,
+ | `dim_6` String,
+ | `dim_7` String,
+ | `dim_8` String,
+ | `dim_9` String,
+ | `dim_10` String,
+ | `dim_11` String,
+ | `dim_12` String,
+ | `dim_13` String,
+ | `dim_14` String,
+ | `dim_15` String,
+ | `dim_16` String,
+ | `dim_17` String,
+ | `dim_18` String,
+ | `dim_19` String,
+ | `dim_20` String,
+ | `dim_21` String,
+ | `dim_22` String,
+ | `dim_23` String,
+ | `dim_24` String,
+ | `dim_25` String,
+ | `dim_26` String,
+ | `dim_27` String,
+ | `dim_28` String,
+ | `dim_29` String,
+ | `dim_30` String,
+ | `dim_31` String,
+ | `dim_32` String,
+ | `dim_33` String,
+ | `dim_34` String,
+ | `dim_35` String,
+ | `dim_36` String,
+ | `dim_37` String,
+ | `dim_38` String,
+ | `dim_39` String,
+ | `dim_40` String,
+ | `dim_41` String,
+ | `dim_42` String,
+ | `dim_43` String,
+ | `dim_44` String,
+ | `dim_45` String,
+ | `dim_46` String,
+ | `dim_47` String,
+ | `dim_48` String,
+ | `dim_49` String,
+ | `dim_50` String,
+ | `dim_52` String,
+ | `dim_53` String,
+ | `dim_54` String,
+ | `dim_55` String,
+ | `dim_56` String,
+ | `dim_57` String,
+ | `dim_58` String,
+ | `dim_59` String,
+ | `dim_60` String,
+ | `dim_61` String,
+ | `dim_62` String,
+ | `dim_63` String,
+ | `dim_64` String,
+ | `dim_65` String,
+ | `dim_66` String,
+ | `dim_67` String,
+ | `dim_68` String,
+ | `dim_69` String,
+ | `dim_70` String,
+ | `dim_71` String,
+ | `dim_72` String,
+ | `dim_73` String,
+ | `dim_74` String,
+ | `dim_75` String,
+ | `dim_76` String,
+ | `dim_77` String,
+ | `dim_78` String,
+ | `dim_79` String,
+ | `dim_80` String,
+ | `dim_81` String,
+ | `dim_82` String,
+ | `dim_83` String,
+ | `dim_84` String,
+ | `dim_85` String,
+ | `dim_86` String,
+ | `dim_87` String,
+ | `dim_88` String,
+ | `dim_89` String,
+ | `dim_90` String,
+ | `dim_91` String,
+ | `dim_92` String,
+ | `dim_93` String,
+ | `dim_94` String,
+ | `dim_95` String,
+ | `dim_96` String,
+ | `dim_97` String,
+ | `dim_98` String,
+ | `dim_99` String,
+ | `dim_100` String,
+ | `counter_1` double,
+ | `counter_2` double,
+ | `counter_3` double,
+ | `counter_4` double,
+ | `counter_5` double,
+ | `counter_6` double,
+ | `counter_7` double,
+ | `counter_8` double,
+ | `counter_9` double,
+ | `counter_10` double,
+ | `counter_11` double,
+ | `counter_12` double,
+ | `counter_13` double,
+ | `counter_14` double,
+ | `counter_15` double,
+ | `counter_16` double,
+ | `counter_17` double,
+ | `counter_18` double,
+ | `counter_19` double,
+ | `counter_20` double,
+ | `counter_21` double,
+ | `counter_22` double,
+ | `counter_23` double,
+ | `counter_24` double,
+ | `counter_25` double,
+ | `counter_26` double,
+ | `counter_27` double,
+ | `counter_28` double,
+ | `counter_29` double,
+ | `counter_30` double,
+ | `counter_31` double,
+ | `counter_32` double,
+ | `counter_33` double,
+ | `counter_34` double,
+ | `counter_35` double,
+ | `counter_36` double,
+ | `counter_37` double,
+ | `counter_38` double,
+ | `counter_39` double,
+ | `counter_40` double,
+ | `counter_41` double,
+ | `counter_42` double,
+ | `counter_43` double,
+ | `counter_44` double,
+ | `counter_45` double,
+ | `counter_46` double,
+ | `counter_47` double,
+ | `counter_48` double,
+ | `counter_49` double,
+ | `counter_50` double,
+ | `counter_51` double,
+ | `counter_52` double,
+ | `counter_53` double,
+ | `counter_54` double,
+ | `counter_55` double,
+ | `counter_56` double,
+ | `counter_57` double,
+ | `counter_58` double,
+ | `counter_59` double,
+ | `counter_60` double,
+ | `counter_61` double,
+ | `counter_62` double,
+ | `counter_63` double,
+ | `counter_64` double,
+ | `counter_65` double,
+ | `counter_66` double,
+ | `counter_67` double,
+ | `counter_68` double,
+ | `counter_69` double,
+ | `counter_70` double,
+ | `counter_71` double,
+ | `counter_72` double,
+ | `counter_73` double,
+ | `counter_74` double,
+ | `counter_75` double,
+ | `counter_76` double,
+ | `counter_77` double,
+ | `counter_78` double,
+ | `counter_79` double,
+ | `counter_80` double,
+ | `counter_81` double,
+ | `counter_82` double,
+ | `counter_83` double,
+ | `counter_84` double,
+ | `counter_85` double,
+ | `counter_86` double,
+ | `counter_87` double,
+ | `counter_88` double,
+ | `counter_89` double,
+ | `counter_90` double,
+ | `counter_91` double,
+ | `counter_92` double,
+ | `counter_93` double,
+ | `counter_94` double,
+ | `counter_95` double,
+ | `counter_96` double,
+ | `counter_97` double,
+ | `counter_98` double,
+ | `counter_99` double,
+ | `counter_100` double,
+ | `batchno` double
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE dim_apn_iot
+ |(
+ | `city_ascription` String,
+ | `industry` String,
+ | `apn_name` String,
+ | `service_level` String,
+ | `customer_name` String,
+ | `id` bigint
+ |)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE tradeflow_all (
+ | m_month smallint,
+ | hs_code string ,
+ | country smallint,
+ | dollar_value double ,
+ | quantity double ,
+ | unit smallint,
+ | b_country smallint,
+ | imex smallint,
+ | y_year smallint)
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE country (
+ | countryid smallint ,
+ | country_en string ,
+ | country_cn string )
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim,
+ s"""
+ |CREATE TABLE updatetime (
+ | countryid smallint ,
+ | imex smallint ,
+ | hs_len smallint ,
+ | minstartdate string ,
+ | startdate string ,
+ | newdate string ,
+ | minnewdate string )
+ |STORED BY 'org.apache.carbondata.format'
+ """.stripMargin.trim
+ )
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
new file mode 100644
index 0000000..3806dac
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/TestSQLBatch.scala
@@ -0,0 +1,584 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv
+
+object TestSQLBatch {
+
+ val testSQLBatch = Seq[String](
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_category in ('Sport', 'Books', 'Home')
+ | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
+ | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year = 1999 + 2
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
+ |FROM date_dim dt, store_sales, item
+ |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.B
+ |FROM
+ | fact
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.B
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ |UNION ALL
+ |SELECT fact.A
+ |FROM
+ | fact
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E AND f1.B = 2
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E AND d1.E = 3
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*) AS A
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim d2 ON (f1.K = d2.K AND d2.E > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim d1 ON (fact.K = d1.K)
+ | JOIN dim d2 ON (fact.K = d2.K AND d2.E > 0)
+ |WHERE fact.E IS NULL AND fact.C > d1.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > dim.E AND (dim.E IS NULL OR dim1.G IS NULL)
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > dim.E OR dim1.G IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E OR dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E AND dim.E IS NULL
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > dim.E
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0 AND dim1.K IS NOT NULL)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K AND fact.K IS NOT NULL)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.E IS NULL AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K AND dim1.G > 0)
+ |WHERE fact.C > fact.E AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND (fact.C > dim.E OR dim1.G > 0)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND fact.C > dim.E OR dim1.G > 0
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E AND fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E OR fact.C > dim.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |WHERE fact.C > fact.E
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,COUNT(*) AS A
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ |--HAVING COUNT(*) > 5
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*)--, my_fun(3) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,COUNT(*) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)
+ | JOIN dim1 ON (fact.K = dim1.K)
+ |GROUP BY fact.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
+ |FROM (SELECT fact.A, fact.B
+ | FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)) FOO
+ |GROUP BY FOO.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT FOO.A, sum(cast(FOO.B as bigint)) AS S
+ |FROM (SELECT fact.A, fact.B
+ | FROM
+ | fact
+ | JOIN dim ON (fact.K = dim.K)) FOO
+ |GROUP BY FOO.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,f1.B,COUNT(*)
+ |FROM
+ | fact f1
+ | JOIN fact f2 ON (f1.K = f2.K)
+ | JOIN fact f3 ON (f1.K = f3.K)
+ |GROUP BY f1.A,f1.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | LEFT OUTER JOIN dim ON (fact.K = dim.K)
+ |GROUP BY fact.A,fact.B
+ """.stripMargin.trim,
+ s"""
+ |SELECT fact.A,fact.B,fact.C,sum(cast(dim.D as bigint)) AS S1
+ |FROM
+ | fact
+ | LEFT OUTER JOIN dim ON (fact.K = dim.K)
+ |GROUP BY fact.A,fact.B,fact.C
+ """.stripMargin.trim,
+// s"""
+// |SELECT *
+// |FROM fact, dim
+// """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | COUNT(*) numsales
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
+ """.stripMargin.trim
+ )
+ val testSQLBatch2 = Seq[String](
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT f1.A,COUNT(*) AS B
+ |FROM
+ | fact f1
+ | JOIN dim d1 ON (f1.K = d1.K)
+ | JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,date_dim.d_date
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, item.i_current_price,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ | SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price)) over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_category in ('Sport', 'Books', 'Home')
+ | AND date_dim.d_date between cast('1999-02-22' as date) AND (cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, item.i_class, item.i_current_price
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, date_dim.d_year, date_dim.d_qoy,
+ | item.i_category, SUM(store_sales.ss_ext_sales_price) ext_sales_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ | AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, item.i_category
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand
+ """.stripMargin.trim,
+ s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand_id brand, SUM(ss_ext_sales_price) ext_price
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manager_id = 28
+ | AND date_dim.d_year = 1999
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT 'store' channel, item.i_brand_id, item.i_class_id, item.i_category_id,
+ | SUM(store_sales.ss_quantity*store_sales.ss_list_price) sales, count(*) number_sales
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND date_dim.d_year = 1999 + 2
+ | AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_class_id,item.i_category_id
+ """.stripMargin.trim,
+ s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk item_sk, dt.d_date solddate, count(*) cnt
+ |FROM date_dim dt, store_sales, item
+ |WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND dt.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), item.i_item_sk,dt.d_date
+ """.stripMargin.trim,
+
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | COUNT(*) numsales
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY store_sales.ss_store_sk,date_dim.d_year GROUPING SETS (store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT store_sales.ss_store_sk,date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY CUBE(store_sales.ss_store_sk,date_dim.d_year)
+ """.stripMargin.trim,
+ s"""
+ |SELECT date_dim.d_moy,date_dim.d_qoy, date_dim.d_year,
+ | SUM(store_sales.ss_ext_sales_price) as itemrevenue
+ |FROM date_dim, store_sales
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |GROUP BY ROLLUP(date_dim.d_moy,date_dim.d_qoy, date_dim.d_year)
+ """.stripMargin.trim
+ )
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
new file mode 100644
index 0000000..2e91e80
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ExtractJoinConditionsSuite.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+import org.apache.spark.sql.catalyst.plans.{Inner, _}
+
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+class ExtractJoinConditionsSuite extends ModularPlanTest {
+ val testRelation0 = LocalRelation('a.int, 'b.int, 'c.int)
+ val testRelation1 = LocalRelation('d.int)
+ val testRelation2 = LocalRelation('b.int,'c.int,'e.int)
+
+ test("join only") {
+ val left = testRelation0.where('a === 1)
+ val right = testRelation1
+ val originalQuery =
+ left.join(right, condition = Some("d".attr === "b".attr || "d".attr === "c".attr)).analyze
+ val modularPlan = originalQuery.modularize
+ val extracted = modularPlan.extractJoinConditions(modularPlan.children(0),modularPlan.children(1))
+
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.LocalRelation(tbl2,_),Inner,Some(cond2)) =>
+ Seq(cond2)
+ }
+
+ compareExpressions(correctAnswer, extracted)
+ }
+
+ test("join and filter") {
+ val left = testRelation0.where('b === 2).subquery('l)
+ val right = testRelation2.where('b === 2).subquery('r)
+ val originalQuery =
+ left.join(right,condition = Some("r.b".attr === 2 && "l.c".attr === "r.c".attr)).analyze
+ val modularPlan = originalQuery.modularize
+ val extracted = modularPlan.extractJoinConditions(modularPlan.children(0),modularPlan.children(1))
+
+ val originalQuery1 =
+ left.join(right,condition = Some("l.c".attr === "r.c".attr)).analyze
+
+ val correctAnswer = originalQuery1 match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.Filter(cond2,logical.LocalRelation(tbl2,_)),Inner,Some(cond3)) =>
+ Seq(cond3)
+ }
+
+ compareExpressions(correctAnswer, extracted)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
new file mode 100644
index 0000000..e80a0cb
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/IsSPJGHSuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.plans.modular.ModularPlan
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+/**
+ * Tests for the isSPJGH function of [[ModularPlan]].
+ */
+class IsSPJGHSuite extends ModularPlanTest {
+ val testRelation0 = LocalRelation('a.int, 'b.int, 'c.int)
+ val testRelation1 = LocalRelation('d.int, 'e.int)
+
+ def assertIsSPJGH(plan: ModularPlan, result: Boolean = true): Unit = {
+ if (plan.isSPJGH != result) {
+ val ps = plan.toString
+ println(s"Plans should return sameResult = $result\n$ps")
+ }
+ }
+
+ test("project only") {
+ assertIsSPJGH(testRelation0.select('a).analyze.modularize)
+ assertIsSPJGH(testRelation0.select('a,'b).analyze.modularize)
+ }
+
+ test("groupby-project") {
+ assertIsSPJGH(testRelation0.select('a).groupBy('a)('a).select('a).analyze.modularize)
+ assertIsSPJGH(testRelation0.select('a,'b).groupBy('a,'b)('a,'b).select('a).analyze.modularize)
+ }
+
+ test("groupby-project-filter") {
+ assertIsSPJGH(testRelation0.where('a === 1).select('a,'b).groupBy('a,'b)('a,'b).select('a).analyze.modularize)
+ }
+
+ test("groupby-project-filter-join") {
+ assertIsSPJGH(testRelation0.where('b === 1).join(testRelation1.where('d === 1),condition = Some("d".attr === "b".attr || "d".attr === "c".attr)).groupBy('b,'c)('b,'c).select('b).analyze.modularize)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
new file mode 100644
index 0000000..e5b6ca5
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/LogicalToModularPlanSuite.scala
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import org.apache.spark.sql.catalyst._
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.aggregate.Count
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.plans.{LeftOuter, RightOuter, _}
+
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.modular.{JoinEdge, ModularRelation}
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+class LogicalToModularPlanSuite extends ModularPlanTest {
+
+ val testRelation0 = LocalRelation('a.int, 'b.int, 'c.int)
+
+ val testRelation1 = LocalRelation('d.int)
+
+ val testRelation2 = LocalRelation('c.int, 'd.int)
+
+ test("select only") {
+ val originalQuery =
+ testRelation0
+ .select('a.attr)
+ .analyze
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Project(proj,logical.LocalRelation(tbl,_)) =>
+ ModularRelation(null,null,tbl,NoFlags,Seq.empty).select(proj:_*)(tbl:_*)()(Map.empty)()
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("select-project-groupby grouping without aggregate function") {
+ val originalQuery =
+ testRelation0
+ .select('a)
+ .groupBy('a)('a)
+ .select('a).analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Project(proj1,logical.Aggregate(grp,agg,logical.Project(proj2,logical.LocalRelation(tbl,_)))) =>
+ ModularRelation(null,null,tbl,NoFlags,Seq.empty).select(proj2:_*)(tbl:_*)()(Map.empty)().groupBy(agg:_*)(proj2:_*)(grp:_*).select(proj1:_*)(proj1:_*)()(Map.empty)()
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("select-project with filter") {
+ val originalQuery =
+ testRelation0
+ .where('a + 'b === 1)
+ .select('a + 'b as 'e)
+ .analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Project(proj,logical.Filter(cond,logical.LocalRelation(tbl,_))) =>
+ ModularRelation(null,null,tbl,NoFlags,Seq.empty).select(proj:_*)(tbl:_*)(cond)(Map.empty)()
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("join") {
+ val left = testRelation0.where('a === 1)
+ val right = testRelation1
+ val originalQuery =
+ left.join(right, condition = Some("d".attr === "b".attr || "d".attr === "c".attr)).analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.LocalRelation(tbl2,_),Inner,Some(cond2)) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty)).select(tbl1++tbl2:_*)(tbl1++tbl2:_*)(Seq(cond1,cond2):_*)(Map.empty)(JoinEdge(0,1,Inner))
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("left outer join") {
+ val left = testRelation0.where('a === 1)
+ val right = testRelation1
+ val originalQuery =
+ left.join(right, LeftOuter, condition = Some("d".attr === "b".attr || "d".attr === "c".attr)).analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.LocalRelation(tbl2,_),LeftOuter,Some(cond2)) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty)).select(tbl1++tbl2:_*)(tbl1++tbl2:_*)(Seq(cond1,cond2):_*)(Map.empty)(JoinEdge(0,1,LeftOuter))
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("right outer join") {
+ val left = testRelation0.where('a === 1)
+ val right = testRelation1
+ val originalQuery =
+ left.join(right, RightOuter, condition = Some("d".attr === "b".attr || "d".attr === "c".attr)).analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.LocalRelation(tbl2,_),RightOuter,Some(cond2)) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty)).select(tbl1++tbl2:_*)(tbl1++tbl2:_*)(Seq(cond1,cond2):_*)(Map.empty)(JoinEdge(0,1,RightOuter))
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("joins: conjunctive predicates #1 with alias") {
+ val left = testRelation0.where('a === 1).subquery('x)
+ val right = testRelation1.subquery('y)
+ val originalQuery =
+ left.join(right, condition = Some("x.b".attr === "y.d".attr)).analyze
+
+ val modularized = analysis.EliminateSubqueryAliases(originalQuery).modularize
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.LocalRelation(tbl2,_),Inner,Some(cond2)) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty)).select(tbl1++tbl2:_*)(tbl1++tbl2:_*)(Seq(cond1,cond2):_*)(Map.empty)(JoinEdge(0,1,Inner))
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("joins: conjunctive predicates #2 with alias") {
+ val lleft = testRelation0.where('a >= 3).subquery('z)
+ val left = testRelation0.where('a === 1).subquery('x)
+ val right = testRelation0.subquery('y)
+ val originalQuery =
+ lleft.join(
+ left.join(right, condition = Some("x.b".attr === "y.b".attr)),
+ condition = Some("z.a".attr === "x.b".attr))
+ .analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Join(logical.Filter(cond1,logical.LocalRelation(tbl1,_)),logical.Join(logical.Filter(cond2,logical.LocalRelation(tbl2,_)),logical.LocalRelation(tbl3,_),Inner,Some(cond3)),Inner,Some(cond4)) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty),ModularRelation(null,null,tbl3,NoFlags,Seq.empty)).select(tbl1++tbl2++tbl3:_*)(tbl1++tbl2++tbl3:_*)(Seq(cond1,cond2,cond3,cond4):_*)(Map.empty)(JoinEdge(0,1,Inner),JoinEdge(1,2,Inner))
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("SPJGH query") {
+ val left = testRelation0.where('b >= 1).subquery('x)
+ val right = testRelation2.where('d >= 2).subquery('y)
+
+ val originalQuery =
+ left.join(right, Inner, Option("x.c".attr ==="y.c".attr))
+ .groupBy("x.a".attr)("x.a".attr as 'f, Count("x.b") as 'g)
+ .select('f)
+ .where('g > 1).analyze
+
+ val modularized = originalQuery.modularize
+ val correctAnswer = originalQuery match {
+ case logical.Project(proj0, logical.Filter(cond1, logical.Project(proj1, logical.Aggregate(grp,agg,logical.Join(logical.Filter(cond2,logical.LocalRelation(tbl1,_)),logical.Filter(cond3,logical.LocalRelation(tbl2,_)),Inner,Some(cond4)))))) =>
+ Seq(ModularRelation(null,null,tbl1,NoFlags,Seq.empty),ModularRelation(null,null,tbl2,NoFlags,Seq.empty)).select(tbl1++tbl2:_*)(tbl1++tbl2:_*)(Seq(cond2,cond3,cond4):_*)(Map.empty)(JoinEdge(0,1,Inner)).groupBy(agg:_*)(tbl1++tbl2:_*)(grp:_*).select(proj0:_*)(proj1:_*)(cond1)(Map.empty)()
+ }
+ comparePlans(modularized, correctAnswer)
+ }
+
+ test("non-SPJGH query") {
+ val mqoAnswer = try testRelation0.where('b > 2).select('a).orderBy('a.asc).analyze.modularize catch {
+ case e: Exception =>
+ s"""
+ |Exception thrown while modularizing query:
+ |== Exception ==
+ |$e
+ """.stripMargin.trim
+ }
+ val correctAnswer =
+ s"""
+ |Exception thrown while modularizing query:
+ |== Exception ==
+ |java.lang.UnsupportedOperationException: unsupported operation: No modular plan for
+ |Sort [a#0 ASC NULLS FIRST], true
+ |+- Project [a#0]
+ | +- Filter (b#1 > 2)
+ | +- LocalRelation <empty>, [a#0, b#1, c#2]
+ """.stripMargin.trim
+ compareMessages(mqoAnswer.toString,correctAnswer)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
new file mode 100644
index 0000000..26f68fe
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/ModularToSQLSuite.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+
+class ModularToSQLSuite extends ModularPlanTest with BeforeAndAfterAll {
+ import org.apache.carbondata.mv.TestSQLBatch._
+
+ override protected def beforeAll(): Unit = {
+ drop
+
+ sql(
+ s"""
+ |CREATE TABLE Fact (
+ | `A` int,
+ | `B` int,
+ | `C` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE Dim (
+ | `D` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE Dim1 (
+ | `F` int,
+ | `G` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE store_sales (
+ | `ss_sold_date_sk` int,
+ | `ss_item_sk` int,
+ | `ss_quantity` int,
+ | `ss_list_price` decimal(7,2),
+ | `ss_ext_sales_price` decimal(7,2),
+ | `ss_store_sk` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE date_dim (
+ | `d_date_sk` int,
+ | `d_date` date,
+ | `d_year` int,
+ | `d_moy` int,
+ | `d_qoy` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE item (
+ | `i_item_sk` int,
+ | `i_item_id` string,
+ | `i_brand` string,
+ | `i_brand_id` int,
+ | `i_item_desc` string,
+ | `i_class_id` int,
+ | `i_class` string,
+ | `i_category` string,
+ | `i_category_id` int,
+ | `i_manager_id` int,
+ | `i_current_price` decimal(7,2)
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sqlContext.udf.register("my_fun", (s: Integer) => s)
+ }
+
+
+ private def drop = {
+ sql(s"drop table if exists Fact")
+ sql(s"drop table if exists Dim")
+ sql(s"drop table if exists Dim1")
+ sql(s"drop table if exists store_sales")
+ sql(s"drop table if exists date_dim")
+ sql(s"drop table if exists item")
+ }
+
+ test("convert modular plans to sqls") {
+ testSQLBatch.foreach { query =>
+ testPlan(query)
+ }
+ }
+
+ private def testPlan(query: String) = {
+ val analyzed = sql(query).queryExecution.analyzed
+ val optimized = analyzed.optimize
+ val modularPlan = analyzed.optimize.modularize
+
+ println(s"\n\n===== ACTUAL QUERY =====\n\n${ query } \n")
+
+ println(s"\n\n===== MODULAR PLAN =====\n\n${ modularPlan.treeString } \n")
+
+ val compactSql = modularPlan.asCompactSQL
+ val convertedSql = modularPlan.asOneLineSQL
+
+ println(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
+
+ val analyzed1 = sql(convertedSql).queryExecution.analyzed
+ val modularPlan1 = analyzed1.optimize.modularize
+
+ println(s"\n\n===== CONVERTED SQL =====\n\n$compactSql \n")
+
+ println(s"\n\n===== MODULAR PLAN1 =====\n\n${ modularPlan1.treeString } \n")
+
+ comparePlans(modularPlan, modularPlan1)
+ }
+
+ override protected def afterAll(): Unit = {
+ drop
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
new file mode 100644
index 0000000..c64826f
--- /dev/null
+++ b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/plans/SignatureSuite.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans
+
+import org.apache.spark.sql.catalyst.util._
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.mv.dsl._
+import org.apache.carbondata.mv.plans.modular.ModularPlanSignatureGenerator
+import org.apache.carbondata.mv.testutil.ModularPlanTest
+import org.apache.carbondata.mv.testutil.Tpcds_1_4_Tables.tpcds1_4Tables
+
+class SignatureSuite extends ModularPlanTest with BeforeAndAfterAll {
+ import org.apache.carbondata.mv.TestSQLBatch._
+
+ override protected def beforeAll(): Unit = {
+ sql("drop database if exists tpcds1 cascade")
+ sql("create database tpcds1")
+ sql("use tpcds1")
+ tpcds1_4Tables.foreach { create_table =>
+ sql(create_table)
+ }
+
+ sql(
+ s"""
+ |CREATE TABLE Fact (
+ | `A` int,
+ | `B` int,
+ | `C` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE Dim (
+ | `D` int,
+ | `E` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sql(
+ s"""
+ |CREATE TABLE Dim1 (
+ | `F` int,
+ | `G` int,
+ | `K` int
+ |)
+ |ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ |STORED AS TEXTFILE
+ """.stripMargin.trim
+ )
+
+ sqlContext.udf.register("my_fun", (s: Integer) => s)
+ }
+
+
+ test("test signature computing") {
+
+ testSQLBatch.foreach { query =>
+ val analyzed = sql(query).queryExecution.analyzed
+ val modularPlan = analyzed.optimize.modularize
+ val sig = ModularPlanSignatureGenerator.generate(modularPlan)
+ sig match {
+ case Some(s) if (s.groupby != true || s.datasets != Set("default.fact","default.dim")) =>
+ println(
+ s"""
+ |=== FAIL: signature do not match ===
+ |${sideBySide(s.groupby.toString, true.toString).mkString("\n")}
+ |${sideBySide(s.datasets.toString, Set("Fact","Dim").toString).mkString("\n")}
+ """.stripMargin)
+ case _ =>
+ }
+ }
+ }
+
+ override protected def afterAll(): Unit = {
+ sql("use default")
+ sql("drop database if exists tpcds1 cascade")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index b851599..b43ae3f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.metadata.ColumnIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchemaStorageProvider}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema, DataMapSchemaStorageProvider}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.processing.exception.DataLoadingException
@@ -601,11 +601,15 @@ object CarbonScalaUtil {
/**
* Create datamap provider using class name
*/
- def createDataMapProvider(className: String, sparkSession: SparkSession,
- storageProvider: DataMapSchemaStorageProvider): Object = {
+ def createDataMapProvider(
+ className: String,
+ sparkSession: SparkSession,
+ table: CarbonTable,
+ schema: DataMapSchema): Object = {
CarbonReflectionUtils.createObject(
className,
+ table,
sparkSession,
- storageProvider)._1.asInstanceOf[Object]
+ schema)._1.asInstanceOf[Object]
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
index 370f80c..2e31a82 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/util/SparkSQLUtil.scala
@@ -17,9 +17,14 @@
package org.apache.spark.sql.util
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.internal.SessionState
object SparkSQLUtil {
def sessionState(sparkSession: SparkSession): SessionState = sparkSession.sessionState
+
+ def execute(logicalPlan: LogicalPlan, sparkSession: SparkSession): DataFrame = {
+ Dataset.ofRows(sparkSession, logicalPlan)
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
index 3fe1f0d..1126b2e 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
@@ -21,12 +21,15 @@ import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandExcept
import org.apache.carbondata.core.datamap.DataMapProvider;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.spark.util.CarbonScalaUtil;
+import static org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.MV;
import static org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.PREAGGREGATE;
import static org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider.TIMESERIES;
import org.apache.spark.sql.SparkSession;
+
public class DataMapManager {
private static DataMapManager INSTANCE;
@@ -50,6 +53,12 @@ public class DataMapManager {
provider = new PreAggregateDataMapProvider(mainTable, dataMapSchema, sparkSession);
} else if (dataMapSchema.getProviderName().equalsIgnoreCase(TIMESERIES.toString())) {
provider = new TimeseriesDataMapProvider(mainTable, dataMapSchema, sparkSession);
+ } else if (dataMapSchema.getProviderName().equalsIgnoreCase(MV.toString())) {
+ provider = (DataMapProvider) CarbonScalaUtil.createDataMapProvider(
+ "org.apache.carbondata.mv.datamap.MVDataMapProvider",
+ sparkSession,
+ mainTable,
+ dataMapSchema);
} else {
provider = new IndexDataMapProvider(mainTable, dataMapSchema, sparkSession);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index cf5a4ae..a27b694 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -176,7 +176,14 @@ case class CarbonDropDataMapCommand(
}
}
} else {
- dropDataMapFromSystemFolder(sparkSession)
+ try {
+ dropDataMapFromSystemFolder(sparkSession)
+ } catch {
+ case e: Exception =>
+ if (!ifExistsSet) {
+ throw e
+ }
+ }
}
Seq.empty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 61df9b1..9576fb1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.execution.command.datamap.CarbonDropDataMapCommand
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7c460e2..7273c76 100644
--- a/pom.xml
+++ b/pom.xml
@@ -636,6 +636,12 @@
<script.exetension>.bat</script.exetension>
</properties>
</profile>
+ <profile>
+ <id>mv</id>
+ <modules>
+ <module>datamap/mv/plan</module>
+ </modules>
+ </profile>
</profiles>
</project>
[4/4] carbondata git commit: [CARBONDATA-2474] Support Modular Plan
for Materialized View DataMap
Posted by ra...@apache.org.
[CARBONDATA-2474] Support Modular Plan for Materialized View DataMap
Currently carbon supports preaggregate datamap, which only supports preaggregate on single table. To improve it, we can add join capability by implementing Materialized View.
In carbon Materialized View, Modular Plan is the basic abstraction for materialized view query plan. In this PR, modular plan module is added, it defines plan tree structure and conversion from Spark Logical Plan.
This closes #2301
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ffddba70
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ffddba70
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ffddba70
Branch: refs/heads/master
Commit: ffddba704bdc110b7fae9054b186c931cfb23744
Parents: 443b717
Author: Jacky Li <ja...@qq.com>
Authored: Sat May 12 17:23:20 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sat May 12 20:31:43 2018 +0530
----------------------------------------------------------------------
.../core/datamap/DataMapStoreManager.java | 36 +-
.../schema/datamap/DataMapClassProvider.java | 3 +-
.../metadata/schema/table/DataMapSchema.java | 4 +-
datamap/mv/plan/pom.xml | 157 +
.../org/apache/carbondata/mv/dsl/package.scala | 101 +
.../mv/expressions/modular/subquery.scala | 169 +
.../mv/plans/modular/AggregatePushDown.scala | 170 +
.../carbondata/mv/plans/modular/Flags.scala | 71 +
.../mv/plans/modular/Harmonizer.scala | 236 +
.../mv/plans/modular/ModularPatterns.scala | 237 +
.../mv/plans/modular/ModularPlan.scala | 206 +
.../modular/ModularPlanSignatureGenerator.scala | 73 +
.../mv/plans/modular/ModularRelation.scala | 143 +
.../mv/plans/modular/Modularizer.scala | 117 +
.../mv/plans/modular/basicOperators.scala | 86 +
.../mv/plans/modular/queryGraph.scala | 24 +
.../apache/carbondata/mv/plans/package.scala | 55 +
.../mv/plans/util/BirdcageOptimizer.scala | 199 +
.../plans/util/Logical2ModularExtractions.scala | 355 ++
.../util/LogicalPlanSignatureGenerator.scala | 101 +
.../carbondata/mv/plans/util/Printers.scala | 347 ++
.../carbondata/mv/plans/util/SQLBuild.scala | 31 +
.../carbondata/mv/plans/util/SQLBuildDSL.scala | 428 ++
.../carbondata/mv/plans/util/SQLBuilder.scala | 262 ++
.../carbondata/mv/plans/util/Signature.scala | 49 +
.../carbondata/mv/plans/util/TableCluster.scala | 55 +
.../mv/testutil/ModularPlanTest.scala | 180 +
.../mv/testutil/Tpcds_1_4_QueryBatch.scala | 4293 ++++++++++++++++++
.../mv/testutil/Tpcds_1_4_Tables.scala | 819 ++++
.../org/apache/carbondata/mv/TestSQLBatch.scala | 584 +++
.../mv/plans/ExtractJoinConditionsSuite.scala | 67 +
.../carbondata/mv/plans/IsSPJGHSuite.scala | 59 +
.../mv/plans/LogicalToModularPlanSuite.scala | 196 +
.../carbondata/mv/plans/ModularToSQLSuite.scala | 164 +
.../carbondata/mv/plans/SignatureSuite.scala | 104 +
.../carbondata/spark/util/CarbonScalaUtil.scala | 12 +-
.../apache/spark/sql/util/SparkSQLUtil.scala | 7 +-
.../carbondata/datamap/DataMapManager.java | 9 +
.../datamap/CarbonDropDataMapCommand.scala | 9 +-
.../command/table/CarbonDropTableCommand.scala | 1 +
pom.xml | 6 +
41 files changed, 10214 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 9f7af2b..a3be26a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -64,7 +64,7 @@ public final class DataMapStoreManager {
/**
* Contains the datamap catalog for each datamap provider.
*/
- private Map<String, DataMapCatalog> dataMapCatalogs = new ConcurrentHashMap<>();
+ private Map<String, DataMapCatalog> dataMapCatalogs = null;
private Map<String, TableSegmentRefresher> segmentRefreshMap = new ConcurrentHashMap<>();
@@ -166,7 +166,8 @@ public final class DataMapStoreManager {
* @param dataMapSchema
*/
public synchronized void registerDataMapCatalog(DataMapProvider dataMapProvider,
- DataMapSchema dataMapSchema) {
+ DataMapSchema dataMapSchema) throws IOException {
+ intializeDataMapCatalogs(dataMapProvider);
String name = dataMapSchema.getProviderName();
DataMapCatalog dataMapCatalog = dataMapCatalogs.get(name);
if (dataMapCatalog == null) {
@@ -185,6 +186,9 @@ public final class DataMapStoreManager {
* @param dataMapSchema
*/
public synchronized void unRegisterDataMapCatalog(DataMapSchema dataMapSchema) {
+ if (dataMapCatalogs == null) {
+ return;
+ }
String name = dataMapSchema.getProviderName();
DataMapCatalog dataMapCatalog = dataMapCatalogs.get(name);
if (dataMapCatalog != null) {
@@ -197,11 +201,37 @@ public final class DataMapStoreManager {
* @param providerName
* @return
*/
- public DataMapCatalog getDataMapCatalog(String providerName) {
+ public DataMapCatalog getDataMapCatalog(DataMapProvider dataMapProvider, String providerName)
+ throws IOException {
+ intializeDataMapCatalogs(dataMapProvider);
return dataMapCatalogs.get(providerName);
}
/**
+ * Initialize by reading all datamaps from store and re register it
+ * @param dataMapProvider
+ */
+ private void intializeDataMapCatalogs(DataMapProvider dataMapProvider) throws IOException {
+ if (dataMapCatalogs == null) {
+ dataMapCatalogs = new ConcurrentHashMap<>();
+ List<DataMapSchema> dataMapSchemas = getAllDataMapSchemas();
+ for (DataMapSchema schema : dataMapSchemas) {
+ DataMapCatalog dataMapCatalog = dataMapCatalogs.get(schema.getProviderName());
+ if (dataMapCatalog == null) {
+ dataMapCatalog = dataMapProvider.createDataMapCatalog();
+ dataMapCatalogs.put(schema.getProviderName(), dataMapCatalog);
+ }
+ try {
+ dataMapCatalog.registerSchema(schema);
+ } catch (Exception e) {
+ // Ignore the schema
+ LOGGER.error(e, "Error while registering schema");
+ }
+ }
+ }
+ }
+
+ /**
* It gives the default datamap of the table. Default datamap of any table is BlockletDataMap
*
* @param table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
index d5a99e8..772bb40 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapClassProvider.java
@@ -30,7 +30,8 @@ public enum DataMapClassProvider {
PREAGGREGATE("org.apache.carbondata.core.datamap.AggregateDataMap", "preaggregate"),
TIMESERIES("org.apache.carbondata.core.datamap.TimeSeriesDataMap", "timeseries"),
LUCENE("org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory","lucene"),
- BLOOMFILTER("org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapFactory", "bloomfilter");
+ BLOOMFILTER("org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapFactory", "bloomfilter"),
+ MV("org.apache.carbondata.core.datamap.MVDataMap", "mv");
/**
* Fully qualified class name of datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
index 611f298..7f6e86f 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -152,7 +152,9 @@ public class DataMapSchema implements Serializable, Writable {
*/
public boolean isIndexDataMap() {
if (providerName.equalsIgnoreCase(DataMapClassProvider.PREAGGREGATE.getShortName()) ||
- providerName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.getShortName())) {
+ providerName.equalsIgnoreCase(DataMapClassProvider.TIMESERIES.getShortName()) ||
+ providerName.equalsIgnoreCase(DataMapClassProvider.MV.getShortName()) ||
+ ctasQuery != null) {
return false;
} else {
return true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/pom.xml b/datamap/mv/plan/pom.xml
new file mode 100644
index 0000000..6a36fc5
--- /dev/null
+++ b/datamap/mv/plan/pom.xml
@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ <relativePath>../../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-mv-plan</artifactId>
+ <name>Apache CarbonData :: Materialized View Plan</name>
+
+ <properties>
+ <dev.path>${basedir}/../../../dev</dev.path>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <testSourceDirectory>src/test/scala</testSourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.18</version>
+ <!-- Note config is repeated in scalatest config -->
+ <configuration>
+ <skip>false</skip>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <argLine>-Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</argLine>
+ <systemProperties>
+ <java.awt.headless>true</java.awt.headless>
+ </systemProperties>
+ <testFailureIgnore>false</testFailureIgnore>
+ <failIfNoTests>false</failIfNoTests>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>2.17</version>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <id>testCompile</id>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <phase>test</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.4.1</version>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>com.ning.maven.plugins</groupId>
+ <artifactId>maven-duplicate-finder-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>1.0</version>
+ <!-- Note config is repeated in surefire config -->
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <testFailureIgnore>false</testFailureIgnore>
+ <filereports>CarbonTestSuite.txt</filereports>
+ <argLine>-ea -Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
+ </argLine>
+ <stderr />
+ <environmentVariables>
+ </environmentVariables>
+ <systemProperties>
+ <java.awt.headless>true</java.awt.headless>
+ </systemProperties>
+ </configuration>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
new file mode 100644
index 0000000..20b5e8a
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv
+
+import scala.language.implicitConversions
+
+import org.apache.spark.sql.catalyst._
+import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.modular.{JoinEdge, ModularPlan}
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.util._
+
+/**
+ * A collection of implicit conversions that create a DSL for constructing data structures
+ * for modular plans.
+ *
+ */
+package object dsl {
+
+ // object plans {
+
+ implicit class DslModularPlan(val modularPlan: ModularPlan) {
+ def select(outputExprs: NamedExpression*)
+ (inputExprs: Expression*)
+ (predicateExprs: Expression*)
+ (aliasMap: Map[Int, String])
+ (joinEdges: JoinEdge*): ModularPlan = {
+ modular
+ .Select(
+ outputExprs,
+ inputExprs,
+ predicateExprs,
+ aliasMap,
+ joinEdges,
+ Seq(modularPlan),
+ NoFlags,
+ Seq.empty,
+ Seq.empty)
+ }
+
+ def groupBy(outputExprs: NamedExpression*)
+ (inputExprs: Expression*)
+ (predicateExprs: Expression*): ModularPlan = {
+ modular
+ .GroupBy(outputExprs, inputExprs, predicateExprs, None, modularPlan, NoFlags, Seq.empty)
+ }
+
+ def harmonize: ModularPlan = modularPlan.harmonized
+ }
+
+ implicit class DslModularPlans(val modularPlans: Seq[ModularPlan]) {
+ def select(outputExprs: NamedExpression*)
+ (inputExprs: Expression*)
+ (predicateList: Expression*)
+ (aliasMap: Map[Int, String])
+ (joinEdges: JoinEdge*): ModularPlan = {
+ modular
+ .Select(
+ outputExprs,
+ inputExprs,
+ predicateList,
+ aliasMap,
+ joinEdges,
+ modularPlans,
+ NoFlags,
+ Seq.empty,
+ Seq.empty)
+ }
+
+ def union(): ModularPlan = modular.Union(modularPlans, NoFlags, Seq.empty)
+ }
+
+ implicit class DslLogical2Modular(val logicalPlan: LogicalPlan) {
+ def resolveonly: LogicalPlan = analysis.SimpleAnalyzer.execute(logicalPlan)
+
+ def modularize: ModularPlan = modular.SimpleModularizer.modularize(logicalPlan).next
+
+ def optimize: LogicalPlan = BirdcageOptimizer.execute(logicalPlan)
+ }
+
+ // }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala
new file mode 100644
index 0000000..cfe341a
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/expressions/modular/subquery.scala
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.expressions.modular
+
+import org.apache.spark.sql.catalyst.expressions.{AttributeSeq, AttributeSet, Expression, ExprId, LeafExpression, NamedExpression, OuterReference, PlanExpression, Predicate, Unevaluable}
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.mv.plans.modular.ModularPlan
+
+/**
+ * A base interface for expressions that contain a [[ModularPlan]].
+ */
+abstract class ModularSubquery(
+ plan: ModularPlan,
+ children: Seq[Expression],
+ exprId: ExprId) extends PlanExpression[ModularPlan] {
+ override lazy val resolved: Boolean = childrenResolved && plan.resolved
+ override lazy val references: AttributeSet =
+ if (plan.resolved) {
+ super.references -- plan.outputSet
+ } else {
+ super.references
+ }
+
+ override def withNewPlan(plan: ModularPlan): ModularSubquery
+
+ override def semanticEquals(o: Expression): Boolean = {
+ o match {
+ case p: ModularSubquery =>
+ this.getClass.getName.equals(p.getClass.getName) && plan.sameResult(p.plan) &&
+ children.length == p.children.length &&
+ children.zip(p.children).forall(p => p._1.semanticEquals(p._2))
+ case _ => false
+ }
+ }
+
+ def canonicalize(attrs: AttributeSeq): ModularSubquery = {
+ // Normalize the outer references in the subquery plan.
+ val normalizedPlan = plan.transformAllExpressions {
+ case OuterReference(r) => OuterReference(QueryPlan.normalizeExprId(r, attrs))
+ }
+ withNewPlan(normalizedPlan).canonicalized.asInstanceOf[ModularSubquery]
+ }
+}
+
+/**
+ * A subquery that will return only one row and one column. This will be converted into a physical
+ * scalar subquery during planning.
+ *
+ * Note: `exprId` is used to have a unique name in explain string output.
+ */
+case class ScalarModularSubquery(
+ plan: ModularPlan,
+ children: Seq[Expression] = Seq.empty,
+ exprId: ExprId = NamedExpression.newExprId)
+ extends ModularSubquery(plan, children, exprId) with Unevaluable {
+ override def dataType: DataType = plan.schema.fields.head.dataType
+
+ override def nullable: Boolean = true
+
+ override def withNewPlan(plan: ModularPlan): ScalarModularSubquery = copy(plan = plan)
+
+ override def toString: String = s"scalar-modular-subquery#${ exprId.id } $conditionString"
+
+ override lazy val canonicalized: Expression = {
+ ScalarModularSubquery(
+ plan.canonicalized,
+ children.map(_.canonicalized),
+ ExprId(0))
+ }
+}
+
+object ScalarModularSubquery {
+ def hasCorrelatedScalarSubquery(e: Expression): Boolean = {
+ e.find {
+ case s: ScalarModularSubquery => s.children.nonEmpty
+ case _ => false
+ }.isDefined
+ }
+}
+
+/**
+ * A [[ListQuery]] expression defines the query which we want to search in an IN subquery
+ * expression. It should and can only be used in conjunction with an IN expression.
+ *
+ * For example (SQL):
+ * {{{
+ * SELECT *
+ * FROM a
+ * WHERE a.id IN (SELECT id
+ * FROM b)
+ * }}}
+ */
+case class ModularListQuery(
+ plan: ModularPlan,
+ children: Seq[Expression] = Seq.empty,
+ exprId: ExprId = NamedExpression.newExprId)
+ extends ModularSubquery(plan, children, exprId) with Unevaluable {
+ override def dataType: DataType = plan.schema.fields.head.dataType
+
+ override def nullable: Boolean = false
+
+ override def withNewPlan(plan: ModularPlan): ModularListQuery = copy(plan = plan)
+
+ override def toString: String = s"modular-list#${ exprId.id } $conditionString"
+
+ override lazy val canonicalized: Expression = {
+ ModularListQuery(
+ plan.canonicalized,
+ children.map(_.canonicalized),
+ ExprId(0))
+ }
+}
+
+/**
+ * The [[Exists]] expression checks if a row exists in a subquery given some correlated condition.
+ *
+ * For example (SQL):
+ * {{{
+ * SELECT *
+ * FROM a
+ * WHERE EXISTS (SELECT *
+ * FROM b
+ * WHERE b.id = a.id)
+ * }}}
+ */
+case class ModularExists(
+ plan: ModularPlan,
+ children: Seq[Expression] = Seq.empty,
+ exprId: ExprId = NamedExpression.newExprId)
+ extends ModularSubquery(plan, children, exprId) with Predicate with Unevaluable {
+ override def nullable: Boolean = false
+
+ override def withNewPlan(plan: ModularPlan): ModularExists = copy(plan = plan)
+
+ override def toString: String = s"modular-exists#${ exprId.id } $conditionString"
+
+ override lazy val canonicalized: Expression = {
+ ModularExists(
+ plan.canonicalized,
+ children.map(_.canonicalized),
+ ExprId(0))
+ }
+}
+
+/**
+ * A place holder for generated SQL for subquery expression.
+ */
+case class SubqueryHolder(override val sql: String) extends LeafExpression with Unevaluable {
+ override def dataType: DataType = NullType
+
+ override def nullable: Boolean = true
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala
new file mode 100644
index 0000000..77efaf7
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/AggregatePushDown.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import scala.collection._
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, Cast, Divide, ExprId, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.aggregate._
+
+trait AggregatePushDown { // self: ModularPlan =>
+
+ def findPushThroughAggregates(outputList: Seq[NamedExpression],
+ selAliasMap: AttributeMap[Attribute],
+ fact: ModularRelation): Map[Int, (NamedExpression, Seq[NamedExpression])] = {
+ var pushable = true
+ val map = scala.collection.mutable.Map[Int, (NamedExpression, Seq[NamedExpression])]()
+ outputList.zipWithIndex.foreach {
+ // TODO: find out if the first two case as follows really needed. Comment out for now.
+ case (attr: Attribute, i) if !fact.outputSet.contains(attr) => pushable = false
+ case (alias: Alias, i)
+ if alias.child.isInstanceOf[Attribute] &&
+ !fact.outputSet.contains(alias.child.asInstanceOf[Attribute]) => pushable = false
+ case (alias: Alias, i) if alias.child.isInstanceOf[AggregateExpression] =>
+ val res = transformAggregate(
+ alias.child
+ .asInstanceOf[AggregateExpression],
+ selAliasMap,
+ i,
+ fact,
+ map,
+ Some((alias.name, alias.exprId)))
+ if (res.isEmpty) {
+ pushable = false
+ }
+ case (agg: AggregateExpression, i) =>
+ val res = transformAggregate(
+ agg,
+ selAliasMap,
+ i,
+ fact,
+ map,
+ None)
+ if (res.isEmpty) {
+ pushable = false
+ }
+ case _ =>
+ }
+ if (!pushable) {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ } else {
+ map
+ }
+ }
+
+ private def transformAggregate(aggregate: AggregateExpression,
+ selAliasMap: AttributeMap[Attribute],
+ ith: Int,
+ fact: ModularRelation,
+ map: scala.collection.mutable.Map[Int, (NamedExpression, Seq[NamedExpression])],
+ aliasInfo: Option[(String, ExprId)]) = {
+ aggregate match {
+ case cnt@AggregateExpression(Count(exprs), _, false, _) if (exprs.length == 1 && exprs(0)
+ .isInstanceOf[Attribute]) =>
+ val tAttr = selAliasMap.get(exprs(0).asInstanceOf[Attribute]).getOrElse(exprs(0))
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val cnt1 = AggregateExpression(Count(tAttr), cnt.mode, false)
+ val alias = Alias(cnt1, cnt1.toString)()
+ val tSum = AggregateExpression(Sum(alias.toAttribute), cnt.mode, false, cnt.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tSum, name)(exprId = id), Seq(alias)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case cnt@AggregateExpression(Count(exprs), _, false, _) if (exprs.length == 1 && exprs(0)
+ .isInstanceOf[Literal]) =>
+ val cnt1 = AggregateExpression(Count(exprs(0)), cnt.mode, false)
+ val alias = Alias(cnt1, cnt1.toString)()
+ val tSum = AggregateExpression(Sum(alias.toAttribute), cnt.mode, false, cnt.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tSum, name)(exprId = id), Seq(alias)))
+ case sum@AggregateExpression(Sum(expr), _, false, _) if (expr.isInstanceOf[Attribute]) =>
+ val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr)
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val sum1 = AggregateExpression(Sum(tAttr), sum.mode, false)
+ val alias = Alias(sum1, sum1.toString)()
+ val tSum = AggregateExpression(Sum(alias.toAttribute), sum.mode, false, sum.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tSum, name)(exprId = id), Seq(alias)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case sum@AggregateExpression(Sum(Cast(expr, dataType, timeZoneId)), _, false, _)
+ if expr.isInstanceOf[Attribute] =>
+ val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr)
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val sum1 = AggregateExpression(Sum(Cast(tAttr, dataType, timeZoneId)), sum.mode, false)
+ val alias = Alias(sum1, sum1.toString)()
+ val tSum = AggregateExpression(Sum(alias.toAttribute), sum.mode, false, sum.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tSum, name)(exprId = id), Seq(alias)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case sum@AggregateExpression(Sum(expr), _, false, _) if (expr.isInstanceOf[Literal]) =>
+ val sum1 = AggregateExpression(Sum(expr), sum.mode, false)
+ val alias = Alias(sum1, sum1.toString)()
+ val tSum = AggregateExpression(Sum(alias.toAttribute), sum.mode, false, sum.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tSum, name)(exprId = id), Seq(alias)))
+ case max@AggregateExpression(Max(expr), _, false, _) if (expr.isInstanceOf[Attribute]) =>
+ val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr)
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val max1 = AggregateExpression(Sum(tAttr), max.mode, false)
+ val alias = Alias(max1, max1.toString)()
+ val tMax = AggregateExpression(Max(alias.toAttribute), max.mode, false, max.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tMax, name)(exprId = id), Seq(alias)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case min@AggregateExpression(Min(expr), _, false, _) if (expr.isInstanceOf[Attribute]) =>
+ val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr)
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val min1 = AggregateExpression(Min(tAttr), min.mode, false)
+ val alias = Alias(min1, min1.toString)()
+ val tMin = AggregateExpression(Max(alias.toAttribute), min.mode, false, min.resultId)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tMin, name)(exprId = id), Seq(alias)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case avg@AggregateExpression(Average(expr), _, false, _) if (expr
+ .isInstanceOf[Attribute]) =>
+ val tAttr = selAliasMap.get(expr.asInstanceOf[Attribute]).getOrElse(expr)
+ .asInstanceOf[Attribute]
+ if (fact.outputSet.contains(tAttr)) {
+ val savg = AggregateExpression(Sum(tAttr), avg.mode, false)
+ val cavg = AggregateExpression(Count(tAttr), avg.mode, false)
+ val sAvg = Alias(savg, savg.toString)()
+ val cAvg = Alias(cavg, cavg.toString)()
+ val tAvg = Divide(sAvg.toAttribute, cAvg.toAttribute)
+ val (name, id) = aliasInfo.getOrElse(("", NamedExpression.newExprId))
+ map += (ith -> (Alias(tAvg, name)(exprId = id), Seq(sAvg, cAvg)))
+ } else {
+ Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ case _ => Map.empty[Int, (NamedExpression, Seq[NamedExpression])]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Flags.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Flags.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Flags.scala
new file mode 100644
index 0000000..2b9d8cf
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Flags.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+trait Flags {
+ // each query allows only one of each of the following keywords
+ final val DISTINCT = 1L << 0
+ final val LIMIT = 1L << 1
+ final val SORT = 1L << 2
+ final val GLOBAL = 1L << 3
+ final val LOCAL = 1L << 4
+ final val EXPAND = 1L << 5
+
+ // to determine each Seq[Expression] in varagrgs belong to which keyword
+ // final val SortLimit = SORT | LIMIT
+
+ def flagToString(flag: Long): String = {
+ flag match {
+ case DISTINCT => "DISTINCT"
+ case LIMIT => "LIMIT"
+ }
+ }
+
+ // List of the raw flags that have expressions as arguments
+ // TODO: add EXPAND
+ private def pickledWithExpressions = {
+ Array[Long](SORT, LIMIT, EXPAND)
+ }
+
+ final val MaxBitPosition = 6
+
+ final val pickledListOrder: List[Long] = {
+ val all = 0 to MaxBitPosition map (1L << _)
+ all.toList filter (pickledWithExpressions contains _)
+ }
+ final val rawFlagPickledOrder: Array[Long] = pickledListOrder.toArray
+
+ type FlagSet = Long
+
+ val NoFlags: FlagSet = 0L
+
+ implicit class FlagSetUtils(var flags: FlagSet) {
+ def hasFlag(mask: Long): Boolean = (flags & mask) != 0L
+
+ def hasFlag(mask: Int): Boolean = hasFlag(mask.toLong)
+
+ def setFlag(mask: Long): FlagSet = { flags |= mask; flags }
+
+ def resetFlag(mask: Long): FlagSet = { flags &= ~mask; flags }
+
+ def initFlags(mask: Long): FlagSet = { flags = mask; flags }
+ }
+
+}
+
+object Flags extends Flags
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Harmonizer.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Harmonizer.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Harmonizer.scala
new file mode 100644
index 0000000..91201a1
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Harmonizer.scala
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.internal.SQLConf
+
+import org.apache.carbondata.mv.plans
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.modular.Flags._
+
+abstract class Harmonizer(conf: SQLConf)
+ extends RuleExecutor[ModularPlan] {
+
+ // protected val fixedPoint = FixedPoint(conf.getConfString("spark.mv.harmonizer
+ // .maxIterations").toInt)
+ protected val fixedPoint = FixedPoint(conf.optimizerMaxIterations)
+
+ def batches: Seq[Batch] = {
+ Batch(
+ "Data Harmonizations", fixedPoint,
+ HarmonizeDimensionTable,
+ HarmonizeFactTable) :: Nil
+ }
+}
+
+/**
+ * An default Harmonizer
+ */
+object DefaultHarmonizer extends DefaultHarmonizer
+
+class DefaultHarmonizer extends Harmonizer(new SQLConf())
+
+object HarmonizeDimensionTable extends Rule[ModularPlan] with PredicateHelper {
+
+ def apply(plan: ModularPlan): ModularPlan = {
+ plan transform {
+ case s@Select(_, _, _, _, jedges, fact :: dims, _, _, _, _) if
+ jedges.forall(e => e.joinType == LeftOuter || e.joinType == Inner) &&
+ fact.isInstanceOf[ModularRelation] &&
+ dims.filterNot(_.isInstanceOf[modular.LeafNode]).nonEmpty &&
+ dims.forall(d => (d.isInstanceOf[ModularRelation] || HarmonizedRelation.canHarmonize(d))) => {
+ var tPullUpPredicates = Seq.empty[Expression]
+ val tChildren = fact :: dims.map {
+ case m: ModularRelation => m
+ case h@GroupBy(
+ _,
+ _,
+ _,
+ _,
+ s1@Select(_, _, _, _, _, dim :: Nil, NoFlags, Nil, Nil, _),
+ NoFlags,
+ Nil, _) if (dim.isInstanceOf[ModularRelation]) => {
+ val rAliasMap = AttributeMap(h.outputList
+ .collect { case a: Alias => (a.child.asInstanceOf[Attribute], a.toAttribute) })
+ val pullUpPredicates = s1.predicateList
+ .map(replaceAlias(_, rAliasMap.asInstanceOf[AttributeMap[Expression]]))
+ if (pullUpPredicates.forall(cond => canEvaluate(cond, h))) {
+ tPullUpPredicates = tPullUpPredicates ++ pullUpPredicates
+ plans.modular.HarmonizedRelation(h.copy(child = s1.copy(predicateList = Nil)))
+ } else {
+ h
+ }
+ }
+ // case _ =>
+ }
+ if (tChildren.forall(_.isInstanceOf[modular.LeafNode])) {
+ s.copy(predicateList = s.predicateList ++ tPullUpPredicates, children = tChildren)
+ } else {
+ s
+ }
+ }
+ // s.withNewChildren(fact :: dims.map { case m: modular.ModularRelation => m; case h
+ // => HarmonizedRelation(h) })}
+ // s.copy(predicateList = predicateList ++ moveUpPredicates, children = tChildren)}
+ // fact :: dims.map { case m: modular.ModularRelation => m; case h => HarmonizedRelation(h)
+ // })}
+ }
+ }
+
+}
+
+object HarmonizeFactTable extends Rule[ModularPlan] with PredicateHelper with AggregatePushDown {
+
+ def apply(plan: ModularPlan): ModularPlan = {
+ plan transform {
+ case g@GroupBy(_, _, _, _,
+ s@Select(_, _, _, aliasm, jedges, fact :: dims, _, _, _, _), _, _, _)
+ if s.adjacencyList.keySet.size <= 1 &&
+ jedges.forall(e => e.joinType == Inner) && // !s.flags.hasFlag(DISTINCT) &&
+ fact.isInstanceOf[ModularRelation] &&
+ (fact :: dims).forall(_.isInstanceOf[modular.LeafNode]) &&
+ dims.nonEmpty => {
+ val selAliasMap = AttributeMap(s.outputList.collect {
+ case a: Alias if (a.child.isInstanceOf[Attribute]) => (a.toAttribute, a.child
+ .asInstanceOf[Attribute])
+ })
+ val aggTransMap = findPushThroughAggregates(
+ g.outputList,
+ selAliasMap,
+ fact.asInstanceOf[ModularRelation])
+
+ val constraintsAttributeSet = dims.flatMap(s.extractEvaluableConditions(_))
+ .map(_.references)
+ .foldLeft(AttributeSet.empty)(_ ++ _)
+ val groupingAttributeSet = g.predicateList.map(_.references)
+ .foldLeft(AttributeSet.empty)(_ ++ _)
+ if (aggTransMap.isEmpty ||
+ // TODO: the following condition is too pessimistic, more work needed using methods
+ // similar to those in trait
+ // QueryPlanConstraints
+ !constraintsAttributeSet.subsetOf(groupingAttributeSet)) {
+ g
+ } else {
+ val starJExprs = dims.flatMap(dim => s.extractJoinConditions(fact, dim)).toSeq
+ val gJAttributes = starJExprs.map(expr => expr.references)
+ .foldLeft(AttributeSet.empty)(_ ++ _).filter(fact.outputSet.contains(_))
+ val fExprs = s.extractEvaluableConditions(fact)
+ val gFAttributes = fExprs.map(expr => expr.references)
+ .foldLeft(AttributeSet.empty)(_ ++ _)
+ .filter(fact.outputSet.contains(_))
+ val gGAttributes = g.predicateList.map(expr => expr.references)
+ .foldLeft(AttributeSet.empty)(_ ++ _).filter(fact.outputSet.contains(_))
+ val gAttributes = (gJAttributes ++ gFAttributes ++ gGAttributes).toSeq
+
+ val oAggregates = aggTransMap.map(_._2).flatMap(_._2).toSeq
+
+ val tAliasMap = (aliasm.get(0) match {
+ case Some(name) => Seq((0, name));
+ case _ => Seq.empty
+ }).toMap
+ val sOutput = (oAggregates.map(_.references).foldLeft(AttributeSet.empty)(_ ++ _) ++
+ AttributeSet(gAttributes)).toSeq
+ val hFactSel = plans.modular
+ .Select(
+ sOutput,
+ fact.output,
+ Seq.empty,
+ tAliasMap,
+ Seq.empty,
+ fact :: Nil,
+ NoFlags,
+ Seq.empty,
+ Seq.empty)
+ val hFact = plans.modular
+ .GroupBy(
+ gAttributes ++ oAggregates,
+ sOutput,
+ gAttributes,
+ None,
+ hFactSel,
+ NoFlags,
+ Seq.empty)
+ val hFactName = s"gen_harmonized_${
+ fact.asInstanceOf[ModularRelation]
+ .databaseName
+ }_${ fact.asInstanceOf[ModularRelation].tableName }"
+ val hAliasMap = (aliasm - 0) + (0 -> hFactName)
+ val hInputList = gAttributes ++ oAggregates.map(_.toAttribute) ++
+ dims.flatMap(_.asInstanceOf[modular.LeafNode].output).toSeq
+ // val hPredicateList = s.predicateList
+ val attrOutputList = s.outputList.filter(expr => (expr.isInstanceOf[Attribute]) ||
+ (expr.isInstanceOf[Alias] &&
+ expr.asInstanceOf[Alias].child
+ .isInstanceOf[Attribute]))
+ val aggOutputList = aggTransMap.values.flatMap(t => t._2)
+ .map { ref =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(hFactName))
+ }
+ val hFactOutputSet = hFact.outputSet
+ // Update the outputlist qualifier
+ val hOutputList = (attrOutputList ++ aggOutputList).map {attr =>
+ attr.transform {
+ case ref: Attribute if hFactOutputSet.contains(ref) =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(hFactName))
+ }
+ }.asInstanceOf[Seq[NamedExpression]]
+
+ // Update the predicate qualifier
+ val hPredList = s.predicateList.map{ pred =>
+ pred.transform {
+ case ref: Attribute if hFactOutputSet.contains(ref) =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(hFactName))
+ }
+ }
+ val hSel = s.copy(
+ outputList = hOutputList,
+ inputList = hInputList,
+ aliasMap = hAliasMap,
+ predicateList = hPredList,
+ children = hFact :: dims)
+ val gOutputList = g.outputList.zipWithIndex
+ .map { case (expr, index) =>
+ if (aggTransMap.keySet.contains(index)) {
+ aggTransMap(index)
+ ._1
+ } else {
+ expr
+ }
+ }
+
+ val wip = g.copy(outputList = gOutputList, inputList = hInputList, child = hSel)
+ wip.transformExpressions {
+ case ref: Attribute if hFactOutputSet.contains(ref) =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(hFactName))
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala
new file mode 100644
index 0000000..c546c6e
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPatterns.scala
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression, PredicateHelper, _}
+import org.apache.spark.sql.catalyst.plans.logical.{Distinct, Limit, LogicalPlan, Window}
+
+import org.apache.carbondata.mv.plans.{Pattern, _}
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.util.{ExtractGroupByModule, ExtractSelectModule, ExtractSelectModuleForWindow, ExtractTableModule, ExtractUnionModule}
+
+object SimpleModularizer extends ModularPatterns {
+ def patterns: Seq[Pattern] = {
+ SelectModule ::
+ GroupByModule ::
+ UnionModule ::
+ DataSourceModule :: Nil
+ }
+
+ override protected def collectPlaceholders(plan: ModularPlan): Seq[(ModularPlan, LogicalPlan)] = {
+ plan.collect {
+ case placeholder@ModularizeLater(logicalPlan) => placeholder -> logicalPlan
+ }
+ }
+
+ override protected def prunePlans(plans: Iterator[ModularPlan]): Iterator[ModularPlan] = {
+ plans
+ // plans.filter(_.collect { case n if n.subqueries.nonEmpty => n }.isEmpty)
+ // TODO: find out why the following stmt not working
+ // plans.filter(_.find { case n if n.subqueries.nonEmpty => true }.isEmpty)
+ }
+
+ override protected def makeupAliasMappings(
+ plans: Iterator[ModularPlan]): Iterator[ModularPlan] = {
+ def makeup(plan: ModularPlan): ModularPlan = {
+ plan transform {
+ case g@GroupBy(_, _, _, _, s@Select(_, _, _, aliasmap, _, children, _, _, _, _), _, _, _) =>
+ val aq = AttributeSet(g.outputList).filter(_.qualifier.nonEmpty)
+ val makeupmap = children.zipWithIndex.flatMap {
+ case (child, i) =>
+ aq.find(child.outputSet.contains(_)).map(_.qualifier).flatten.map((i, _))
+ }.toMap
+ g.copy(child = s.copy(aliasMap = makeupmap ++ aliasmap))
+ }
+ }
+
+ plans.map(makeup)
+ }
+}
+
+abstract class ModularPattern extends GenericPattern[ModularPlan] {
+
+ override protected def modularizeLater(plan: LogicalPlan): ModularPlan = ModularizeLater(plan)
+}
+
+case class ModularizeLater(plan: LogicalPlan) extends LeafNode {
+ override def output: Seq[Attribute] = plan.output
+}
+
+abstract class ModularPatterns extends Modularizer[ModularPlan] {
+
+ // self: MQOContext#SparkyModeler =>
+
+ object SelectModule extends Pattern with PredicateHelper {
+
+ private[this] def makeSelectModule(
+ output: Seq[NamedExpression],
+ input: Seq[Expression],
+ predicate: Seq[Expression],
+ aliasmap: Map[Int, String],
+ joinedge: Seq[JoinEdge],
+ flags: FlagSet,
+ children: Seq[ModularPlan],
+ flagSpec: Seq[Seq[Any]],
+ windowSpec: Seq[Seq[Any]]) = {
+ Seq(Select(
+ output,
+ input,
+ predicate,
+ aliasmap,
+ joinedge,
+ children,
+ flags,
+ flagSpec,
+ windowSpec))
+ }
+
+ def apply(plan: LogicalPlan): Seq[ModularPlan] = {
+ plan match {
+ case Distinct(
+ ExtractSelectModule(output, input, predicate, aliasmap, joinedge, children, flags1,
+ fspec1, wspec)) =>
+ val flags = flags1.setFlag(DISTINCT)
+ makeSelectModule(output, input, predicate, aliasmap, joinedge, flags,
+ children.map(modularizeLater), fspec1, wspec)
+
+ case Limit(
+ limitExpr,
+ Distinct(
+ ExtractSelectModule(output, input, predicate, aliasmap, joinedge, children,
+ flags1, fspec1, wspec))) =>
+ val flags = flags1.setFlag(DISTINCT).setFlag(LIMIT)
+ makeSelectModule(output, input, predicate, aliasmap, joinedge, flags,
+ children.map(modularizeLater), Seq(Seq(limitExpr)) ++ fspec1, wspec)
+
+ case Limit(
+ limitExpr,
+ ExtractSelectModule(output, input, predicate, aliasmap, joinedge, children, flags1,
+ fspec1, wspec)) =>
+ val flags = flags1.setFlag(LIMIT)
+ makeSelectModule(output, input, predicate, aliasmap, joinedge, flags,
+ children.map(modularizeLater), Seq(Seq(limitExpr)) ++ fspec1, wspec)
+
+ case ExtractSelectModule(output, input, predicate, aliasmap, joinedge, children, flags1,
+ fspec1, wspec) =>
+ makeSelectModule(output, input, predicate, aliasmap, joinedge, flags1,
+ children.map(modularizeLater), fspec1, wspec)
+
+ case Window(exprs, _, _,
+ ExtractSelectModuleForWindow(output, input, predicate, aliasmap, joinedge, children,
+ flags1, fspec1, wspec)) =>
+ val sel1 = makeSelectModule(output, input, predicate, aliasmap, joinedge, flags1,
+ children.map(modularizeLater), fspec1, wspec)
+ makeSelectModule(
+ output.map(_.toAttribute),
+ output.map(_.toAttribute),
+ Seq.empty,
+ Map.empty,
+ Seq.empty,
+ NoFlags,
+ sel1,
+ Seq.empty,
+ Seq(Seq(exprs)) ++ wspec)
+
+ case _ => Nil
+ }
+ }
+ }
+
+ object GroupByModule extends Pattern with PredicateHelper {
+
+ private[this] def makeGroupByModule(
+ output: Seq[NamedExpression],
+ input: Seq[Expression],
+ predicate: Seq[Expression],
+ flags: FlagSet,
+ alias: Option[String],
+ child: ModularPlan,
+ fspec: Seq[Seq[Any]]) = {
+ val groupby = Some(GroupBy(output, input, predicate, alias, child, flags, fspec))
+ groupby.map(Seq(_)).getOrElse(Nil)
+ }
+
+ def apply(plan: LogicalPlan): Seq[ModularPlan] = {
+ plan match {
+ case Limit(
+ limitExpr,
+ ExtractGroupByModule(output, input, predicate, alias, child, flags1, fspec1)) =>
+ val flags = flags1.setFlag(LIMIT)
+ makeGroupByModule(
+ output,
+ input,
+ predicate,
+ flags,
+ alias,
+ modularizeLater(child),
+ Seq(Seq(limitExpr)) ++ fspec1)
+ case ExtractGroupByModule(output, input, predicate, alias, child, flags1, fspec1) =>
+ makeGroupByModule(output, input, predicate, flags1, alias, modularizeLater(child), fspec1)
+ case _ => Nil
+ }
+ }
+ }
+
+ object UnionModule extends Pattern with PredicateHelper {
+
+ private[this] def makeUnionModule(
+ flags: FlagSet,
+ children: Seq[ModularPlan],
+ fspec: Seq[Seq[Any]]) = {
+ Seq(modular.Union(children, flags, fspec))
+ }
+
+ def apply(plan: LogicalPlan): Seq[ModularPlan] = {
+ plan match {
+ case Distinct(ExtractUnionModule(children, flags1, fspec1)) =>
+ val flags = flags1.setFlag(DISTINCT)
+ makeUnionModule(flags, children.map(modularizeLater), fspec1)
+ case Limit(limitExpr, Distinct(ExtractUnionModule(children, flags1, fspec1))) =>
+ val flags = flags1.setFlag(DISTINCT).setFlag(LIMIT)
+ makeUnionModule(flags, children.map(modularizeLater), Seq(Seq(limitExpr)) ++ fspec1)
+ case Limit(limitExpr, ExtractUnionModule(children, flags1, fspec1)) =>
+ val flags = flags1.setFlag(LIMIT)
+ makeUnionModule(flags, children.map(modularizeLater), Seq(Seq(limitExpr)) ++ fspec1)
+ case ExtractUnionModule(children, flags1, fspec1) =>
+ makeUnionModule(flags1, children.map(modularizeLater), fspec1)
+ case _ => Nil
+ }
+ }
+ }
+
+ object DataSourceModule extends Pattern with Flags with PredicateHelper {
+
+ private[this] def makeDataSourceModule(
+ databaseName: String,
+ tableName: String,
+ output: Seq[NamedExpression],
+ flags: FlagSet,
+ fspec: Seq[Seq[Any]]) = {
+ Seq(ModularRelation(databaseName, tableName, output, flags, fspec))
+ }
+
+ def apply(plan: LogicalPlan): Seq[ModularPlan] = {
+ plan match {
+ case ExtractTableModule(databaseName, tableName, output, Nil, flags1, fspec1) =>
+ makeDataSourceModule(databaseName, tableName, output, flags1, fspec1)
+ case _ => Nil
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlan.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlan.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlan.scala
new file mode 100644
index 0000000..c66df19
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlan.scala
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import scala.collection._
+import scala.collection.mutable.{HashMap, MultiMap}
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.{Expression, PredicateHelper}
+import org.apache.spark.sql.catalyst.plans.{JoinType, QueryPlan}
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
+import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.internal.SQLConf
+
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.util.{Printers, Signature, SQLBuilder}
+
+abstract class ModularPlan
+ extends QueryPlan[ModularPlan]
+ with AggregatePushDown
+ with Logging
+ with Serializable
+ with PredicateHelper with Printers {
+
+ /**
+ * the first two are to support sub-query expressions
+ */
+
+ lazy val resolved: Boolean = expressions.forall(_.resolved) && childrenResolved
+
+ def childrenResolved: Boolean = children.forall(_.resolved)
+
+ private var statsCache: Option[Statistics] = None
+
+ final def stats(spark: SparkSession, conf: SQLConf): Statistics = {
+ statsCache.getOrElse {
+ statsCache = Some(computeStats(spark, conf))
+ statsCache.get
+ }
+ }
+
+ final def invalidateStatsCache(): Unit = {
+ statsCache = None
+ children.foreach(_.invalidateStatsCache())
+ }
+
+ protected def computeStats(spark: SparkSession, conf: SQLConf): Statistics = {
+ // spark.conf.set("spark.sql.cbo.enabled", true)
+ val sqlStmt = asOneLineSQL
+ val plan = spark.sql(sqlStmt).queryExecution.optimizedPlan
+ plan.stats(conf)
+ }
+
+ override def fastEquals(other: TreeNode[_]): Boolean = {
+ this.eq(other)
+ }
+
+ private var _rewritten: Boolean = false
+
+ /**
+ * Marks this plan as already rewritten.
+ */
+ private[mv] def setRewritten(): ModularPlan = {
+ _rewritten = true
+ children.foreach(_.setRewritten())
+ this
+ }
+
+ /**
+ * Returns true if this node and its children have already been gone through query rewrite.
+ * Note this this is only an optimization used to avoid rewriting trees that have already
+ * been rewritten, and can be reset by transformations.
+ */
+ def rewritten: Boolean = {
+ _rewritten
+ }
+
+ private var _skip: Boolean = false
+
+ private[mv] def setSkip(): ModularPlan = {
+ _skip = true
+ children.foreach(_.setSkip())
+ this
+ }
+
+ private[mv] def resetSkip(): ModularPlan = {
+ _skip = false
+ children.foreach(_.resetSkip())
+ this
+ }
+
+ def skip: Boolean = _skip
+
+ def isSPJGH: Boolean = {
+ this match {
+ case modular.Select(_, _, _, _, _,
+ Seq(modular.GroupBy(_, _, _, _,
+ sel_c2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if sel_c2.children.forall(_.isInstanceOf[modular.LeafNode]) => true
+
+ case modular.GroupBy(_, _, _, _, sel_c2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)
+ if sel_c2.children.forall(_.isInstanceOf[modular.LeafNode]) => true
+
+ case modular.Select(_, _, _, _, _, children, _, _, _, _)
+ if children.forall(_.isInstanceOf[modular.LeafNode]) => true
+
+ case _ => false
+ }
+ }
+
+ def signature: Option[Signature] = ModularPlanSignatureGenerator.generate(this)
+
+ def createMutableAdjacencyList(
+ edges: Seq[JoinEdge]
+ ): mutable.HashMap[Int, mutable.Set[(Int, JoinType)]] with mutable.MultiMap[Int, (Int, JoinType)]
+ = {
+ val mm = new HashMap[Int, mutable.Set[(Int, JoinType)]] with MultiMap[Int, (Int, JoinType)]
+ for (edge <- edges) { mm.addBinding(edge.left, (edge.right, edge.joinType)) }
+ mm
+ }
+
+ def createImmutableAdjacencyList(edges: Seq[JoinEdge]): Predef.Map[Int, Seq[(Int, JoinType)]] = {
+ edges.groupBy { _.left }.map { case (k, v) => (k, v.map(e => (e.right, e.joinType))) }
+ }
+
+ def adjacencyList: Map[Int, Seq[(Int, JoinType)]] = Map.empty
+
+ def extractJoinConditions(left: ModularPlan, right: ModularPlan): Seq[Expression] = Seq.empty
+
+ def extractRightEvaluableConditions(left: ModularPlan, right: ModularPlan): Seq[Expression] =
+ Seq.empty
+
+ def extractEvaluableConditions(plan: ModularPlan): Seq[Expression] = Seq.empty
+
+ def asCompactSQL: String = asCompactString(new SQLBuilder(this).fragmentExtract)
+
+ def asOneLineSQL: String = asOneLineString(new SQLBuilder(this).fragmentExtract)
+
+ // for plan without sub-query expression only
+ def asOneLineSQL(subqueryPrefix: String): String = {
+ asOneLineString(new SQLBuilder(
+ this,
+ subqueryPrefix).fragmentExtract)
+ }
+
+ /**
+ * Returns a plan where a best effort attempt has been made to transform `this` in a way
+ * that preserves the result but replaces harmonized dimension table with HarmonizedRelation
+ * and fact table with sub-plan that pre-aggregates the table before join with dimension table
+ *
+ * Some nodes should overwrite this to provide proper harmonization logic.
+ */
+ lazy val harmonized: ModularPlan = DefaultHarmonizer.execute(preHarmonized)
+
+ /**
+ * Do some simple transformation on this plan before harmonizing. Implementations can override
+ * this method to provide customized harmonize logic without rewriting the whole logic.
+ *
+ * We assume queries need to be harmonized are of the form:
+ *
+ * FACT (left) join (harmonized) DIM1 (left) join (harmonized) DIM2 ...
+ *
+ * For queries of not this form, customize this method for them to conform this form.
+ */
+ protected def preHarmonized: ModularPlan = {
+ this
+ }
+}
+
+object ModularPlan extends PredicateHelper {
+
+}
+
+abstract class LeafNode extends ModularPlan {
+ override def children: Seq[ModularPlan] = Nil
+}
+
+abstract class UnaryNode extends ModularPlan {
+ def child: ModularPlan
+
+ override def children: Seq[ModularPlan] = child :: Nil
+}
+
+abstract class BinaryNode extends ModularPlan {
+ def left: ModularPlan
+
+ def right: ModularPlan
+
+ override def children: Seq[ModularPlan] = Seq(left, right)
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlanSignatureGenerator.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlanSignatureGenerator.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlanSignatureGenerator.scala
new file mode 100644
index 0000000..86cae36
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularPlanSignatureGenerator.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.util.{Signature, SignatureGenerator, SignatureRule}
+
+object ModularPlanSignatureGenerator extends SignatureGenerator[ModularPlan] {
+ lazy val rule: SignatureRule[ModularPlan] = ModularPlanRule
+
+ override def generate(plan: ModularPlan): Option[Signature] = {
+ if (plan.isSPJGH) {
+ super.generate(plan)
+ } else {
+ None
+ }
+ }
+}
+
+object ModularPlanRule extends SignatureRule[ModularPlan] {
+
+ def apply(plan: ModularPlan, childSignatures: Seq[Option[Signature]]): Option[Signature] = {
+
+ plan match {
+ case modular.Select(_, _, _, _, _, _, _, _, _, _) =>
+ if (childSignatures.map { _.getOrElse(Signature()).groupby }.forall(x => !x)) {
+ Some(Signature(
+ groupby = false,
+ childSignatures.flatMap { _.getOrElse(Signature()).datasets.toSeq }.toSet))
+ } else if (childSignatures.length == 1 &&
+ childSignatures(0).getOrElse(Signature()).groupby) {
+ childSignatures(0)
+ } else {
+ None
+ }
+ case modular.GroupBy(_, _, _, _, _, _, _, _) =>
+ if (childSignatures.length == 1 && !childSignatures(0).getOrElse(Signature()).groupby) {
+ Some(Signature(groupby = true, childSignatures(0).getOrElse(Signature()).datasets))
+ } else {
+ None
+ }
+ case HarmonizedRelation(source) =>
+ source.signature match {
+ case Some(s) =>
+ Some(Signature(groupby = false, s.datasets))
+ case _ =>
+ None
+ }
+ case modular.ModularRelation(dbname, tblname, _, _, _) =>
+ if (dbname != null && tblname != null) {
+ Some(Signature(groupby = false, Set(Seq(dbname, tblname).mkString("."))))
+ } else {
+ Some(Signature(groupby = false, Set(plan.toString())))
+ }
+ case _ => None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
new file mode 100644
index 0000000..dec0c48
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/ModularRelation.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.JoinType
+import org.apache.spark.sql.catalyst.plans.logical
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
+import org.apache.spark.sql.internal.SQLConf
+
+import org.apache.carbondata.mv.plans.modular.Flags._
+
+object ModularRelation {
+ def apply(outputList: NamedExpression*): ModularRelation = {
+ new ModularRelation(
+ "test",
+ "test",
+ outputList,
+ NoFlags,
+ Seq.empty)
+ }
+}
+
+case class ModularRelation(databaseName: String,
+ tableName: String,
+ outputList: Seq[NamedExpression],
+ flags: FlagSet,
+ rest: Seq[Seq[Any]]) extends LeafNode {
+ override def computeStats(spark: SparkSession, conf: SQLConf): Statistics = {
+ val plan = spark.table(s"${ databaseName }.${ tableName }").queryExecution.optimizedPlan
+ val stats = plan.stats(conf)
+ val output = outputList.map(_.toAttribute)
+ val mapSeq = plan.collect { case n: logical.LeafNode => n }.map {
+ table => AttributeMap(table.output.zip(output))
+ }
+ val rewrites = mapSeq(0)
+ val attributeStats = AttributeMap(stats.attributeStats.iterator
+ .map { pair => (rewrites(pair._1), pair._2) }.toSeq)
+ Statistics(stats.sizeInBytes, stats.rowCount, attributeStats, stats.hints)
+ }
+
+ override def output: Seq[Attribute] = outputList.map(_.toAttribute)
+
+ override def adjacencyList: Map[Int, Seq[(Int, JoinType)]] = Map.empty
+
+ override def equals(that: Any): Boolean = {
+ that match {
+ case that: ModularRelation =>
+ if ((databaseName != null && tableName != null && databaseName == that.databaseName &&
+ tableName == that.tableName) ||
+ (databaseName == null && tableName == null && that.databaseName == null &&
+ that.tableName == null && output.toString == (that.output).toString)) {
+ true
+ } else {
+ false
+ }
+ case _ => false
+ }
+ }
+}
+
+object HarmonizedRelation {
+ def canHarmonize(source: ModularPlan): Boolean = {
+ source match {
+ case g@GroupBy(
+ _,
+ _,
+ _,
+ _,
+ Select(_, _, _, _, _, dim :: Nil, NoFlags, Nil, Nil, _),
+ NoFlags,
+ Nil, _) if (dim.isInstanceOf[ModularRelation]) =>
+ if (g.outputList
+ .forall(col => col.isInstanceOf[AttributeReference] ||
+ (col.isInstanceOf[Alias] &&
+ col.asInstanceOf[Alias].child.isInstanceOf[AttributeReference]))) {
+ true
+ } else {
+ false
+ }
+ case _ => false
+ }
+ }
+}
+
+// support harmonization for dimension table
+case class HarmonizedRelation(source: ModularPlan) extends LeafNode {
+ require(HarmonizedRelation.canHarmonize(source), "invalid plan for harmonized relation")
+ lazy val tableName = source.asInstanceOf[GroupBy].child.children(0).asInstanceOf[ModularRelation]
+ .tableName
+ lazy val databaseName = source.asInstanceOf[GroupBy].child.children(0)
+ .asInstanceOf[ModularRelation].databaseName
+
+ // override def computeStats(spark: SparkSession, conf: SQLConf): Statistics = source.stats
+ // (spark, conf)
+ override def computeStats(spark: SparkSession, conf: SQLConf): Statistics = {
+ val plan = spark.table(s"${ databaseName }.${ tableName }").queryExecution.optimizedPlan
+ val stats = plan.stats(conf)
+ val output = source.asInstanceOf[GroupBy].child.children(0).asInstanceOf[ModularRelation]
+ .outputList.map(_.toAttribute)
+ val mapSeq = plan.collect { case n: logical.LeafNode => n }.map {
+ table => AttributeMap(table.output.zip(output))
+ }
+ val rewrites = mapSeq(0)
+ val aliasMap = AttributeMap(
+ source.asInstanceOf[GroupBy].outputList.collect {
+ case a: Alias if (a.child.isInstanceOf[Attribute]) => (a.child.asInstanceOf[Attribute], a
+ .toAttribute)
+ })
+ val aStatsIterator = stats.attributeStats.iterator.map { pair => (rewrites(pair._1), pair._2) }
+ val attributeStats = AttributeMap(aStatsIterator
+ .map(pair => ((aliasMap.get(pair._1)).getOrElse(pair._1), pair._2)).toSeq)
+
+ Statistics(stats.sizeInBytes, None, attributeStats, stats.hints)
+ }
+
+ override def output: Seq[Attribute] = source.output
+
+ // two harmonized modular relations are equal only if orders of output columns of
+ // their source plans are exactly the same
+ override def equals(that: Any): Boolean = {
+ that match {
+ case that: HarmonizedRelation => source.sameResult(that.source)
+ case _ => false
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Modularizer.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Modularizer.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Modularizer.scala
new file mode 100644
index 0000000..d255359
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/Modularizer.scala
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.expressions.{Exists, ListQuery, ScalarSubquery}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.trees.TreeNode
+
+import org.apache.carbondata.mv.expressions.modular._
+import org.apache.carbondata.mv.plans._
+
+abstract class GenericPattern[TreeType <: TreeNode[TreeType]] extends Logging {
+ protected def modularizeLater(plan: LogicalPlan): TreeType
+
+ def apply(plan: LogicalPlan): Seq[TreeType]
+}
+
+abstract class Modularizer[TreeType <: TreeNode[TreeType]] {
+ def patterns: Seq[GenericPattern[TreeType]]
+
+ // protected def modularizeLater(plan: LogicalPlan) = this.modularize(plan).next()
+
+ def modularize(plan: LogicalPlan): Iterator[TreeType] = {
+ val replaced = plan.transformAllExpressions {
+ case s: ScalarSubquery =>
+ if (s.children.isEmpty) {
+ ScalarModularSubquery(
+ modularize(s.plan).next.asInstanceOf[ModularPlan],
+ s.children,
+ s.exprId)
+ } else {
+ throw new UnsupportedOperationException(s"Expression $s doesn't canonicalized")
+ }
+ case l: ListQuery =>
+ if (l.children.isEmpty) {
+ ModularListQuery(modularize(l.plan).next.asInstanceOf[ModularPlan], l.children, l.exprId)
+ } else {
+ throw new UnsupportedOperationException(s"Expression $l doesn't canonicalized")
+ }
+ case e: Exists =>
+ if (e.children.isEmpty) {
+ ModularExists(modularize(e.plan).next.asInstanceOf[ModularPlan], e.children, e.exprId)
+ } else {
+ throw new UnsupportedOperationException(s"Expression $e doesn't canonicalized")
+ }
+ case o => o
+ }
+ // val replaced = plan
+ val mplans = modularizeCore(replaced)
+ makeupAliasMappings(mplans)
+ }
+
+ private def modularizeCore(plan: LogicalPlan): Iterator[TreeType] = {
+ // Collect modular plan candidates.
+ val candidates = patterns.iterator.flatMap(_ (plan))
+
+ // The candidates may contain placeholders marked as [[modularizeLater]],
+ // so try to replace them by their child plans.
+ val plans = candidates.flatMap { candidate =>
+ val placeholders = collectPlaceholders(candidate)
+
+ if (placeholders.isEmpty) {
+ // Take the candidate as is because it does not contain placeholders.
+ Iterator(candidate)
+ } else {
+ // Plan the logical plan marked as [[modularizeLater]] and replace the placeholders.
+ placeholders.iterator.foldLeft(Iterator(candidate)) {
+ case (candidatesWithPlaceholders, (placeholder, logicalPlan)) =>
+ // Modularize the logical plan for the placeholder
+ val childPlans = this.modularizeCore(logicalPlan)
+
+ candidatesWithPlaceholders.flatMap { candidateWithPlaceholder =>
+ childPlans.map { childPlan =>
+ // Replace the placeholder by the child plan
+ candidateWithPlaceholder.transformUp {
+ case p if p == placeholder => childPlan
+ }
+ }
+ }
+ }
+ }
+ }
+
+ val pruned = prunePlans(plans)
+ // val iter = patterns.view.flatMap(_(plan)).toIterator
+ supports(
+ pruned.hasNext,
+ s"Modular plan not supported (e.g. has subquery expression) for \n$plan")
+ // makeupAliasMappings(pruned)
+ pruned
+ }
+
+ /** Collects placeholders marked as [[modularizeLater]] by pattern and its [[LogicalPlan]]s */
+ protected def collectPlaceholders(plan: TreeType): Seq[(TreeType, LogicalPlan)]
+
+ /** Prunes bad plans to prevent combinatorial explosion. */
+ protected def prunePlans(plans: Iterator[TreeType]): Iterator[TreeType]
+
+ protected def makeupAliasMappings(plans: Iterator[TreeType]): Iterator[TreeType]
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/basicOperators.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/basicOperators.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/basicOperators.scala
new file mode 100644
index 0000000..0a76da1
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/basicOperators.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, _}
+import org.apache.spark.sql.catalyst.plans.JoinType
+
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.modular.Flags._
+
+private[mv] trait Matchable extends ModularPlan {
+ def outputList: Seq[NamedExpression]
+
+ def predicateList: Seq[Expression]
+}
+
+case class GroupBy(
+ outputList: Seq[NamedExpression],
+ inputList: Seq[Expression],
+ predicateList: Seq[Expression],
+ alias: Option[String],
+ child: ModularPlan,
+ flags: FlagSet,
+ flagSpec: Seq[Seq[Any]],
+ dataMapTableRelation: Option[ModularPlan] = None) extends UnaryNode with Matchable {
+ override def output: Seq[Attribute] = outputList.map(_.toAttribute)
+}
+
+case class Select(
+ outputList: Seq[NamedExpression],
+ inputList: Seq[Expression],
+ predicateList: Seq[Expression],
+ aliasMap: Map[Int, String],
+ joinEdges: Seq[JoinEdge],
+ children: Seq[ModularPlan],
+ flags: FlagSet,
+ flagSpec: Seq[Seq[Any]],
+ windowSpec: Seq[Seq[Any]],
+ dataMapTableRelation: Option[ModularPlan] = None) extends ModularPlan with Matchable {
+ override def output: Seq[Attribute] = outputList.map(_.toAttribute)
+
+ override def adjacencyList: scala.collection.immutable.Map[Int, Seq[(Int, JoinType)]] = {
+ joinEdges.groupBy { _.left }.map { case (k, v) => (k, v.map(e => (e.right, e.joinType))) }
+ }
+
+ override def extractJoinConditions(
+ left: ModularPlan, right: ModularPlan): Seq[Expression] = {
+ predicateList.filter(p => p.references.intersect(left.outputSet).nonEmpty &&
+ p.references.intersect(right.outputSet).nonEmpty &&
+ p.references.subsetOf(left.outputSet ++ right.outputSet))
+ }
+
+ override def extractRightEvaluableConditions(
+ left: ModularPlan, right: ModularPlan): Seq[Expression] = {
+ predicateList.filter(p => p.references.subsetOf(left.outputSet ++ right.outputSet) &&
+ p.references.intersect(right.outputSet).nonEmpty)
+ }
+
+ override def extractEvaluableConditions(plan: ModularPlan): Seq[Expression] = {
+ predicateList.filter(p => canEvaluate(p, plan))
+ }
+}
+
+case class Union(children: Seq[ModularPlan], flags: FlagSet, flagSpec: Seq[Seq[Any]])
+ extends ModularPlan {
+ override def output: Seq[Attribute] = children.head.output
+}
+
+case object OneRowTable extends LeafNode {
+ override def output: Seq[Attribute] = Nil
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/queryGraph.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/queryGraph.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/queryGraph.scala
new file mode 100644
index 0000000..da6bc15
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/modular/queryGraph.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.modular
+
+import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.sql.catalyst.plans.JoinType
+
+@DeveloperApi
+case class JoinEdge(left: Int, right: Int, joinType: JoinType)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/package.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/package.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/package.scala
new file mode 100644
index 0000000..8c799fe
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/package.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv
+
+import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.sql.catalyst.expressions.{AttributeSet, Expression, PredicateHelper}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.mv.plans.modular.ModularPlan
+import org.apache.carbondata.mv.plans.util.{CheckSPJG, LogicalPlanSignatureGenerator, Signature}
+
+/**
+ * A a collection of common abstractions for query plans as well as
+ * a base semantic plan representation.
+ */
+package object plans {
+ @DeveloperApi
+ type Pattern = org.apache.carbondata.mv.plans.modular.ModularPattern
+
+ implicit class LogicalPlanUtils(val plan: LogicalPlan) {
+ lazy val isSPJG: Boolean = CheckSPJG.isSPJG(plan)
+ lazy val signature: Option[Signature] = LogicalPlanSignatureGenerator.generate(plan)
+ }
+
+ implicit class MorePredicateHelper(p: PredicateHelper) {
+ def canEvaluate(expr: Expression, plan: ModularPlan): Boolean = {
+ expr.references.subsetOf(plan.outputSet)
+ }
+
+ def canEvaluate(expr: Expression, exprList: Seq[Expression]): Boolean = {
+ expr.references.subsetOf(AttributeSet(exprList))
+ }
+ }
+
+ def supports(supported: Boolean, message: Any) {
+ if (!supported) {
+ throw new UnsupportedOperationException(s"unsupported operation: $message")
+ }
+ }
+}
[3/4] carbondata git commit: [CARBONDATA-2474] Support Modular Plan
for Materialized View DataMap
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/BirdcageOptimizer.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/BirdcageOptimizer.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/BirdcageOptimizer.scala
new file mode 100644
index 0000000..6363089
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/BirdcageOptimizer.scala
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import org.apache.spark.sql.catalyst.analysis._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.optimizer._
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, _}
+import org.apache.spark.sql.catalyst.rules.{RuleExecutor, _}
+import org.apache.spark.sql.internal.SQLConf
+
+object BirdcageOptimizer extends RuleExecutor[LogicalPlan] {
+
+ val conf = new SQLConf()
+ .copy(SQLConf.CASE_SENSITIVE -> true, SQLConf.STARSCHEMA_DETECTION -> true)
+ protected val fixedPoint = FixedPoint(conf.optimizerMaxIterations)
+
+ def batches: Seq[Batch] = {
+ // Technically some of the rules in Finish Analysis are not optimizer rules and belong more
+ // in the analyzer, because they are needed for correctness (e.g. ComputeCurrentTime).
+ // However, because we also use the analyzer to canonicalized queries (for view definition),
+ // we do not eliminate subqueries or compute current time in the analyzer.
+ Batch(
+ "Finish Analysis", Once,
+ EliminateSubqueryAliases,
+ EliminateView,
+ ReplaceExpressions,
+ ComputeCurrentTime,
+ // GetCurrentDatabase(sessionCatalog),
+ RewriteDistinctAggregates,
+ ReplaceDeduplicateWithAggregate) ::
+ //////////////////////////////////////////////////////////////////////////////////////////
+ // Optimizer rules start here
+ //////////////////////////////////////////////////////////////////////////////////////////
+ // - Do the first call of CombineUnions before starting the major Optimizer rules,
+ // since it can reduce the number of iteration and the other rules could add/move
+ // extra operators between two adjacent Union operators.
+ // - Call CombineUnions again in Batch("Operator Optimizations"),
+ // since the other rules might make two separate Unions operators adjacent.
+ Batch(
+ "Union", Once,
+ CombineUnions) ::
+ Batch(
+ "Pullup Correlated Expressions", Once,
+ PullupCorrelatedPredicates) ::
+ Batch(
+ "Subquery", Once,
+ OptimizeSubqueries) ::
+ Batch(
+ "Replace Operators", fixedPoint,
+ ReplaceIntersectWithSemiJoin,
+ ReplaceExceptWithAntiJoin,
+ ReplaceDistinctWithAggregate) ::
+ Batch(
+ "Aggregate", fixedPoint,
+ RemoveLiteralFromGroupExpressions,
+ RemoveRepetitionFromGroupExpressions) ::
+ Batch(
+ "Operator Optimizations", fixedPoint, Seq(
+ // Operator push down
+ PushProjectionThroughUnion,
+ ReorderJoin(conf),
+ EliminateOuterJoin(conf),
+ PushPredicateThroughJoin,
+ PushDownPredicate,
+ // LimitPushDown(conf),
+ ColumnPruning,
+ // InferFiltersFromConstraints(conf),
+ // Operator combine
+ CollapseRepartition,
+ CollapseProject,
+ CollapseWindow,
+ CombineFilters,
+ CombineLimits,
+ CombineUnions,
+ // Constant folding and strength reduction
+ NullPropagation(conf),
+ FoldablePropagation,
+ // OptimizeIn(conf),
+ ConstantFolding,
+ ReorderAssociativeOperator,
+ LikeSimplification,
+ BooleanSimplification,
+ SimplifyConditionals,
+ RemoveDispensableExpressions,
+ SimplifyBinaryComparison,
+ // PruneFilters(conf),
+ EliminateSorts,
+ SimplifyCasts,
+ SimplifyCaseConversionExpressions,
+ RewriteCorrelatedScalarSubquery,
+ EliminateSerialization,
+ RemoveRedundantAliases,
+ RemoveRedundantProject,
+ SimplifyCreateStructOps,
+ SimplifyCreateArrayOps,
+ SimplifyCreateMapOps) ++
+ extendedOperatorOptimizationRules: _*) ::
+ Batch(
+ "Check Cartesian Products", Once,
+ CheckCartesianProducts(conf)) ::
+ // Batch("Join Reorder", Once,
+ // CostBasedJoinReorder(conf)) ::
+ // Batch("Decimal Optimizations", fixedPoint,
+ // DecimalAggregates(conf)) ::
+ Batch(
+ "Object Expressions Optimization", fixedPoint,
+ EliminateMapObjects,
+ CombineTypedFilters) ::
+ // Batch("LocalRelation", fixedPoint,
+ // ConvertToLocalRelation,
+ // PropagateEmptyRelation) ::
+ Batch(
+ "OptimizeCodegen", Once,
+ OptimizeCodegen(conf)) ::
+ Batch(
+ "RewriteSubquery", Once,
+ RewritePredicateSubquery,
+ CollapseProject) :: Nil
+ }
+
+ /**
+ * Optimize all the subqueries inside expression.
+ */
+ object OptimizeSubqueries extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = {
+ plan transformAllExpressions {
+ case s: SubqueryExpression =>
+ val Subquery(newPlan) = BirdcageOptimizer.this.execute(Subquery(s.plan))
+ s.withNewPlan(newPlan)
+ }
+ }
+ }
+
+ /**
+ * Override to provide additional rules for the operator optimization batch.
+ */
+ def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] = {
+ Nil
+ }
+}
+
+/**
+ * Push Aggregate through join to fact table.
+ * Pushes down [[Aggregate]] operators where the `grouping` and `aggregate` expressions can
+ * be evaluated using only the attributes of the fact table, the left or right side of a
+ * star-join.
+ * Other [[Aggregate]] expressions stay in the original [[Aggregate]].
+ *
+ * Check 'Aggregate Pushdown Over Join: Design & Preliminary Results' by LiTao for more details
+ */
+// case class PushAggregateThroughJoin(conf: SQLConf) extends Rule[LogicalPlan] with
+// PredicateHelper {
+//
+// val tableCluster = {
+// val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+// val tableClusterString = conf.getConfString("spark.mv.tableCluster")
+// mapper.readValue(tableClusterString, classOf[TableCluster])
+// }
+//
+// def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+//
+// // Push down aggregate expressions through Join
+// case a @ Aggregate(grouping, aggregate, Project(projectList, Join(left, right, jt, cond)))
+// if (left.isInstanceOf[LeafNode] && => {
+// val fTables: Set[String] = tableCluster.getFact
+// val dTables: Set[String] = tableCluster.getDimension
+// // if canPushThrough(left,a)
+//
+// if (fTables.contains(s"${left.databaseName}.${left.tableName}")
+// Aggregate(newGrouping, newAggregate, Project(projectList, Join(Aggregate(_,_,Project
+// (projectList1, left)), right, jt, cond)))
+// }
+// }
+//
+// private def canPushThrough(join: Join): Boolean = join match {
+// case Join(left : LeafNode, right: LeafNode, Inner, EqualTo(l: AttributeReference,
+// r: AttributeReference)) => true
+//
+//
+// }
+//
+//
+// }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Logical2ModularExtractions.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Logical2ModularExtractions.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Logical2ModularExtractions.scala
new file mode 100644
index 0000000..de65e37
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Logical2ModularExtractions.scala
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference,
+ AttributeSet, Expression, NamedExpression, PredicateHelper}
+import org.apache.spark.sql.catalyst.plans._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.modular.JoinEdge
+
+/**
+ * SelectModule is extracted from logical plan of SPJG query. All join conditions
+ * filter, and project operators in a single Aggregate-less subtree of logical plan
+ * are collected.
+ *
+ * The returned values for this match are as follows:
+ * - Conditions for equi-join
+ * - Conditions for filter
+ * - Project list for project
+ *
+ */
+object ExtractSelectModule extends PredicateHelper {
+ type ReturnType = (Seq[NamedExpression], Seq[Expression], Seq[Expression], Map[Int, String],
+ Seq[JoinEdge], Seq[LogicalPlan], FlagSet, Seq[Seq[Any]], Seq[Seq[Any]])
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = {
+ val (outputs, inputs, predicates, joinedges, children, isSelect, _, flags, fspecs, wspecs) =
+ collectProjectsFiltersJoinsAndSort(plan)
+ if (!isSelect) {
+ None
+ } else {
+ Some(
+ outputs,
+ inputs,
+ predicates,
+ collectChildAliasMappings(
+ AttributeSet(outputs).toSeq ++ AttributeSet(predicates).toSeq,
+ children),
+ joinedges,
+ children,
+ flags,
+ fspecs,
+ wspecs)
+ }
+ }
+
+ def collectProjectsFiltersJoinsAndSort(plan: LogicalPlan): (Seq[NamedExpression],
+ Seq[Expression], Seq[Expression], Seq[JoinEdge], Seq[LogicalPlan], Boolean, Map[Attribute,
+ Expression], FlagSet, Seq[Seq[Any]], Seq[Seq[Any]]) = {
+ plan match {
+ case Project(fields, child) =>
+ val (_, inputs, predicates, joinedges, children, _, aliases, flags, fspecs, wspecs) =
+ collectProjectsFiltersJoinsAndSort(child)
+ val substitutedFields = fields.map(substitute(aliases)).asInstanceOf[Seq[NamedExpression]]
+ (substitutedFields, inputs, predicates, joinedges, children, true, collectAliases(
+ substitutedFields), flags, fspecs, wspecs)
+
+ case Filter(condition, child) =>
+ val (outputs, inputs, predicates, joinedges, children, _, aliases, flags, fspecs, wspecs)
+ = collectProjectsFiltersJoinsAndSort(child)
+ val substitutedCondition = substitute(aliases)(condition)
+ (outputs, inputs, predicates.flatMap(splitConjunctivePredicates) ++
+ splitConjunctivePredicates(substitutedCondition), joinedges, children,
+ true, aliases, flags, fspecs, wspecs)
+
+ case Sort(order, global, child) =>
+ val (outputs, inputs, predicates, joinedges, children, _, aliases, flags, fspecs, wspecs)
+ = collectProjectsFiltersJoinsAndSort(child)
+ val substitutedOrder = order.map(substitute(aliases))
+ (outputs, inputs, predicates, joinedges, children, true, aliases, if (global) {
+ flags.setFlag(SORT).setFlag(GLOBAL)
+ } else {
+ flags.setFlag(SORT)
+ }, Seq(Seq(order)) ++ fspecs, wspecs)
+
+ case Join(left, right, joinType, condition) =>
+ val (loutputs, linputs, lpredicates, ljoinedges, lchildren, _, laliases, lflags, lfspecs,
+ lwspecs) = collectProjectsFiltersJoinsAndSort(left)
+ val (routputs, rinputs, rpredicates, rjoinedges, rchildren, _, raliases, rflags, rfspecs,
+ rwspecs) = collectProjectsFiltersJoinsAndSort(right)
+ val (lcondition, rcondition, ccondition) = split(condition, lchildren, rchildren)
+ val joinEdge = collectJoinEdge(ccondition, lchildren, rchildren, joinType)
+ val adjustedJoinEdges = rjoinedges
+ .map(e => JoinEdge(e.left + lchildren.size, e.right + lchildren.size, e.joinType))
+ val output: Seq[Attribute] = {
+ joinType match {
+ case LeftSemi =>
+ left.output
+ case LeftOuter =>
+ left.output ++ right.output.map(_.withNullability(true))
+ case RightOuter =>
+ left.output.map(_.withNullability(true)) ++ right.output
+ case FullOuter =>
+ left.output.map(_.withNullability(true)) ++ right.output.map(_.withNullability(true))
+ case LeftAnti =>
+ left.output
+ case _ =>
+ left.output ++ right.output
+ }
+ }
+ if (lfspecs.isEmpty && rfspecs.isEmpty && lflags == NoFlags && rflags == NoFlags &&
+ lwspecs.isEmpty && rwspecs.isEmpty) {
+ (output, (linputs ++ rinputs), lpredicates.flatMap(splitConjunctivePredicates) ++
+ rpredicates.flatMap(splitConjunctivePredicates) ++
+ lcondition ++ rcondition ++ ccondition, ljoinedges ++
+ joinEdge ++
+ adjustedJoinEdges,
+ lchildren ++ rchildren, true, laliases ++ raliases, NoFlags, Seq.empty, Seq.empty)
+ } else {
+ throw new UnsupportedOperationException(
+ s"unsupported join: \n left child ${ left } " +
+ s"\n right child ${ right }")
+ }
+
+ case other =>
+ (other.output, other.output, Nil, Nil, Seq(other), false, Map.empty, NoFlags, Seq.empty, Seq
+ .empty)
+ }
+ }
+
+ def collectAliases(fields: Seq[Expression]): Map[Attribute, Expression] = {
+ fields.collect {
+ case a@Alias(child, _) => a.toAttribute -> child
+ }.toMap
+ }
+
+ def substitute(aliases: Map[Attribute, Expression])(expr: Expression): Expression = {
+ expr.transform {
+ case a@Alias(ref: AttributeReference, name) =>
+ aliases.get(ref).map(Alias(_, name)(a.exprId, a.qualifier)).getOrElse(a)
+
+ case a: AttributeReference =>
+ aliases.get(a).map(Alias(_, a.name)(a.exprId, a.qualifier)).getOrElse(a)
+ }
+ }
+
+ def collectChildAliasMappings(attributeSet: Seq[Attribute], children: Seq[LogicalPlan]
+ ): Map[Int, String] = {
+ val aq = attributeSet.filter(_.qualifier.nonEmpty)
+ children.zipWithIndex.flatMap {
+ case (child, i) =>
+ aq.find(child.outputSet.contains(_)).map(_.qualifier).flatten.map((i, _))
+ }.toMap
+ }
+
+ def split(condition: Option[Expression],
+ lchildren: Seq[LogicalPlan],
+ rchildren: Seq[LogicalPlan]): (Seq[Expression], Seq[Expression], Seq[Expression]) = {
+ val left = lchildren.map(_.outputSet).foldLeft(AttributeSet(Set.empty))(_ ++ _)
+ val right = rchildren.map(_.outputSet).foldLeft(AttributeSet(Set.empty))(_ ++ _)
+ val conditions = condition.map(splitConjunctivePredicates).getOrElse(Nil)
+ val (leftEvaluationCondition, rest) = conditions.partition(_.references subsetOf left)
+ val (rightEvaluationCondition, commonCondition) = rest.partition(_.references subsetOf right)
+ (leftEvaluationCondition, rightEvaluationCondition, commonCondition)
+ }
+
+ /*
+ * collectJoinEdge only valid when condition are common condition of above split, left and
+ * right children correspond
+ * to respective two children parameters of above split
+ *
+ */
+ def collectJoinEdge(condition: Seq[Expression],
+ lchildren: Seq[LogicalPlan],
+ rchildren: Seq[LogicalPlan],
+ joinType: JoinType): Seq[JoinEdge] = {
+ val common = condition.map(_.references).foldLeft(AttributeSet(Set.empty))(_ ++ _)
+ val lIdxSeq = lchildren
+ .collect { case x if x.outputSet.intersect(common).nonEmpty => lchildren.indexOf(x) }
+ val rIdxSeq = rchildren
+ .collect { case x if x.outputSet.intersect(common).nonEmpty => rchildren.indexOf(x) +
+ lchildren.size
+ }
+ for (l <- lIdxSeq; r <- rIdxSeq) yield {
+ JoinEdge(l, r, joinType)
+ }
+ }
+
+}
+
+object ExtractSelectModuleForWindow extends PredicateHelper {
+ type ReturnType = (Seq[NamedExpression], Seq[Expression], Seq[Expression], Map[Int, String],
+ Seq[JoinEdge], Seq[LogicalPlan], FlagSet, Seq[Seq[Any]], Seq[Seq[Any]])
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = {
+ collectSelectFromWindowChild(plan)
+ }
+
+ def collectSelectFromWindowChild(plan: LogicalPlan): Option[(Seq[NamedExpression],
+ Seq[Expression], Seq[Expression], Map[Int, String], Seq[JoinEdge], Seq[LogicalPlan], FlagSet,
+ Seq[Seq[Any]], Seq[Seq[Any]])] = {
+ plan match {
+ case agg@Aggregate(_, _, _) =>
+ Some(
+ agg.aggregateExpressions,
+ agg.child.output,
+ Seq.empty,
+ Map.empty,
+ Seq.empty,
+ Seq(agg),
+ NoFlags,
+ Seq.empty,
+ Seq.empty)
+ case ExtractSelectModule(
+ output,
+ input,
+ predicate,
+ aliasmap,
+ joinedge,
+ children,
+ flags,
+ fspec,
+ wspec) =>
+ Some(output, input, predicate, aliasmap, joinedge, children, flags, fspec, wspec)
+ case Window(exprs, _, _, child) =>
+ val ret: Option[(Seq[NamedExpression], Seq[Expression], Seq[Expression], Map[Int,
+ String], Seq[JoinEdge], Seq[LogicalPlan], FlagSet, Seq[Seq[Any]], Seq[Seq[Any]])] =
+ collectSelectFromWindowChild(
+ child)
+ ret.map(r => (r._1, r._2, r._3, r._4, r._5, r._6, r._7, r._8, Seq(Seq(exprs)) ++ r._9))
+ case other => None
+ }
+ }
+}
+
+/**
+ * GroupByModule is extracted from the Aggregate node of logical plan.
+ * The groupingExpressions, aggregateExpressions are collected.
+ *
+ * The returned values for this match are as follows:
+ * - Grouping attributes for the Aggregate node.
+ * - Aggregates for the Aggregate node.
+ * - Project list for project
+ *
+ */
+
+object ExtractGroupByModule extends PredicateHelper {
+ type ReturnType = (Seq[NamedExpression], Seq[Expression], Seq[Expression], Option[String],
+ LogicalPlan, FlagSet, Seq[Seq[Any]])
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = {
+ plan match {
+ case a@logical.Aggregate(_, _, e@Expand(_, _, p: Project)) if isGroupingSet(a, e, p) =>
+ // Assumption: Aggregate's groupingExpressions is composed of
+ // 1) the grouping attributes
+ // 2) gid, which is always the last one
+ val g = a.groupingExpressions.map(_.asInstanceOf[Attribute])
+ val numOriginalOutput = e.output.size - g.size
+ Some(
+ a.aggregateExpressions,
+ e.output,
+ a.groupingExpressions,
+ None,
+ p,
+ NoFlags.setFlag(EXPAND),
+ Seq(Seq(e.projections, e.output, numOriginalOutput)))
+ case logical.Aggregate(groupingExpressions, aggregateExpressions, child) =>
+ Some(
+ aggregateExpressions,
+ child.output,
+ groupingExpressions,
+ None,
+ child,
+ NoFlags,
+ Seq.empty)
+ case other => None
+ }
+ }
+
+ private def isGroupingSet(a: Aggregate, e: Expand, p: Project): Boolean = {
+ assert(a.child == e && e.child == p)
+
+ if (a.groupingExpressions.forall(_.isInstanceOf[Attribute])) {
+ val g = a.groupingExpressions.map(_.asInstanceOf[Attribute])
+ sameOutput(
+ e.output.drop(e.output.size - g.size),
+ a.groupingExpressions.map(_.asInstanceOf[Attribute]))
+ } else {
+ false
+ }
+ }
+
+ private def sameOutput(output1: Seq[Attribute], output2: Seq[Attribute]): Boolean = {
+ output1.size == output2.size &&
+ output1.zip(output2).forall(pair => pair._1.semanticEquals(pair._2))
+ }
+}
+
+object ExtractUnionModule extends PredicateHelper {
+ type ReturnType = (Seq[LogicalPlan], FlagSet, Seq[Seq[Any]])
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = {
+ plan match {
+ case u: Union =>
+ val children = collectUnionChildren(u)
+ Some(children, NoFlags, Seq.empty)
+ case _ => None
+ }
+ }
+
+ private def collectUnionChildren(plan: LogicalPlan): List[LogicalPlan] = {
+ plan match {
+ case Union(children) => children.toList match {
+ case head :: Nil => collectUnionChildren(head)
+ case head :: tail => collectUnionChildren(head) ++ collectUnionChildren(Union(tail))
+ case Nil => Nil
+ }
+ case other => other :: Nil
+ }
+ }
+}
+
+object ExtractTableModule extends PredicateHelper {
+ type ReturnType = (String, String, Seq[NamedExpression], Seq[LogicalPlan], FlagSet, Seq[Seq[Any]])
+
+ def unapply(plan: LogicalPlan): Option[ReturnType] = {
+ plan match {
+ // uncomment for cloudera1 version
+// case m: CatalogRelation =>
+// Some(m.tableMeta.database, m.tableMeta.identifier.table, m.output, Nil, NoFlags,
+// Seq.empty)
+// uncomment for apache version
+ case m: HiveTableRelation =>
+ Some(m.tableMeta.database, m.tableMeta.identifier.table, m.output, Nil, NoFlags,
+ Seq.empty)
+ case l: LogicalRelation =>
+ val tableIdentifier = l.catalogTable.map(_.identifier)
+ val database = tableIdentifier.map(_.database).flatten.getOrElse(null)
+ val table = tableIdentifier.map(_.table).getOrElse(null)
+ Some(database, table, l.output, Nil, NoFlags, Seq.empty)
+ case l: LocalRelation => // used for unit test
+ Some(null, null, l.output, Nil, NoFlags, Seq.empty)
+ case _ => None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
new file mode 100644
index 0000000..0c5661e
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+import org.apache.carbondata.mv.plans._
+
+object CheckSPJG {
+
+ def isSPJG(subplan: LogicalPlan): Boolean = {
+ subplan match {
+ case a: Aggregate =>
+ a.child.collect {
+ case Join(_, _, _, _) | Project(_, _) | Filter(_, _) |
+// CatalogRelation(_, _, _) |
+ LogicalRelation(_, _, _) | LocalRelation(_, _) => true
+ case _ => false
+ }.forall(identity)
+ case _ => false
+ }
+ }
+}
+
+object LogicalPlanSignatureGenerator extends SignatureGenerator[LogicalPlan] {
+ lazy val rule: SignatureRule[LogicalPlan] = LogicalPlanRule
+
+ override def generate(plan: LogicalPlan): Option[Signature] = {
+ if ( plan.isSPJG ) {
+ super.generate(plan)
+ } else {
+ None
+ }
+ }
+}
+
+object LogicalPlanRule extends SignatureRule[LogicalPlan] {
+
+ def apply(plan: LogicalPlan, childSignatures: Seq[Option[Signature]]): Option[Signature] = {
+
+ plan match {
+ case LogicalRelation(_, _, _) =>
+ // TODO: implement this (link to BaseRelation)
+ None
+// case CatalogRelation(tableMeta, _, _) =>
+// Some(Signature(false,
+// Set(Seq(tableMeta.database, tableMeta.identifier.table).mkString("."))))
+ case l: LocalRelation =>
+ // LocalRelation is for unit test cases
+ Some(Signature(groupby = false, Set(l.toString())))
+ case Filter(_, _) =>
+ if (childSignatures.length == 1 && !childSignatures(0).getOrElse(Signature()).groupby) {
+ // if (!childSignatures(0).getOrElse(Signature()).groupby) {
+ childSignatures(0)
+ // }
+ } else {
+ None
+ }
+ case Project(_, _) =>
+ if ( childSignatures.length == 1 && !childSignatures(0).getOrElse(Signature()).groupby ) {
+ childSignatures(0)
+ } else {
+ None
+ }
+ case Join(_, _, _, _) =>
+ if ( childSignatures.length == 2 &&
+ !childSignatures(0).getOrElse(Signature()).groupby &&
+ !childSignatures(1).getOrElse(Signature()).groupby ) {
+ Some(Signature(false,
+ childSignatures(0).getOrElse(Signature()).datasets
+ .union(childSignatures(1).getOrElse(Signature()).datasets)))
+ } else {
+ None
+ }
+ case Aggregate(_, _, _) =>
+ if ( childSignatures.length == 1 && !childSignatures(0).getOrElse(Signature()).groupby ) {
+ Some(Signature(true, childSignatures(0).getOrElse(Signature()).datasets))
+ } else {
+ None
+ }
+ case _ => None
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Printers.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Printers.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Printers.scala
new file mode 100644
index 0000000..b7641d5
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Printers.scala
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import java.io.{OutputStream, PrintWriter, StringWriter}
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression, _}
+import org.apache.spark.sql.catalyst.plans.JoinType
+import org.apache.spark.sql.catalyst.util.quoteIdentifier
+
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.util.SQLBuildDSL._
+
+// scalastyle:off println
+trait Printers {
+
+ abstract class FragmentPrinter(out: PrintWriter) {
+ protected var indentMargin = 0
+ protected val indentStep = 2
+ protected var indentString = " " // 40
+
+ def indent(): Unit = indentMargin += indentStep
+
+ def undent(): Unit = indentMargin -= indentStep
+
+ def println(): Unit = {
+ out.println()
+ while (indentMargin > indentString.length()) {
+ indentString += indentString
+ }
+ if (indentMargin > 0) {
+ out.write(indentString, 0, indentMargin)
+ }
+ }
+
+ def printSeq[a](ls: List[a])(printelem: a => Unit)(printsep: => Unit) {
+ ls match {
+ case List() =>
+ case List(x) => printelem(x)
+ case x :: rest => printelem(x); printsep; printSeq(rest)(printelem)(printsep)
+ }
+ }
+
+ def printColumn(ts: List[Fragment], start: String, sep: String, end: String) {
+ print(start)
+ indent
+ println()
+ printSeq(ts) { print(_) } { print(sep); println() }
+ undent
+ println()
+ print(end)
+ }
+
+ def printRow(ts: List[Fragment], start: String, sep: String, end: String) {
+ print(start)
+ printSeq(ts) { print(_) } { print(sep) }
+ print(end)
+ }
+
+ def printRow(ts: List[Fragment], sep: String) { printRow(ts, "", sep, "") }
+
+ def printFragment(frag: Fragment): Unit
+
+ def print(args: Any*): Unit = {
+ args foreach {
+ case frag: Fragment =>
+ printFragment(frag)
+ case arg =>
+ out.print(if (arg == null) {
+ "null"
+ } else {
+ arg.toString
+ })
+ }
+ }
+ }
+
+ /**
+ * A sql fragment printer which is stingy about vertical whitespace and unnecessary
+ * punctuation.
+ */
+ class SQLFragmentCompactPrinter(out: PrintWriter) extends FragmentPrinter(out) {
+
+ object Position extends Enumeration {
+ type Position = Value
+ val Select, From, GroupBy, Where, Sort, Limit = Value
+ }
+
+ import Position._
+
+ def printSelect(select: Seq[NamedExpression], flags: FlagSet): Unit = {
+ if (flags.hasFlag(DISTINCT)) {
+ print("SELECT DISTINCT %s "
+ .format(select.map(_.sql) mkString ", "))
+ } else {
+ print("SELECT %s ".format(select.map(_.sql) mkString ", "))
+ }
+ }
+
+ def printFromElem(f: (SQLBuildDSL.Fragment, Option[JoinType], Seq[Expression])): Unit = {
+ f._2 map { t => print("%s JOIN ".format(t.sql)) }
+ printFragment(f._1)
+ if (f._3.nonEmpty) {
+ print(" ON %s".format(f._3.map(_.sql).mkString(" AND ")))
+ }
+ }
+
+ def printFrom(
+ from: Seq[(SQLBuildDSL.Fragment, Option[JoinType], Seq[Expression])]): Seq[Unit] = {
+ def fromIndented(
+ x: (SQLBuildDSL.Fragment, Option[JoinType], Seq[Expression])): Unit = {
+ indent
+ println()
+ printFromElem(x)
+ undent
+ }
+
+ print("FROM")
+ from map fromIndented
+ }
+
+ def printWhere(where: Seq[Expression]): Unit = {
+ if (where != Nil) {
+ print("WHERE")
+ // if (where.nonEmpty) {
+ // val condition = where(0)
+ // val dt = condition.dataType
+ // val t = condition.sql
+ // print(t)
+ // }
+ indent
+ println()
+ print("%s".format(where.map(_.sql).mkString(" AND ")))
+ undent
+ }
+ }
+
+ def printGroupby(groupby: (Seq[Expression], Seq[Seq[Expression]])): Unit = {
+
+ if (groupby._1.nonEmpty) {
+ print("GROUP BY %s".format(groupby._1.map(_.sql).mkString(", ")))
+ if (groupby._2.nonEmpty) {
+ print(" GROUPING SETS(%s)"
+ .format(groupby._2.map(e => s"(${ e.map(_.sql).mkString(", ") })").mkString(", ")))
+ }
+ }
+ }
+
+ def printHaving(having: Seq[Expression]): Unit = {
+ if (having != Nil) {
+ print("HAVING %s".format(having.map(_.sql).mkString(" AND ")))
+ }
+ }
+
+ def printUnion(sqls: Seq[Fragment]): Unit = {
+ sqls match {
+ case sql1 :: sql2 :: rest =>
+ printFragment(sql1)
+ println()
+ print("UNION ALL")
+ println()
+ printUnion(sql2 :: rest)
+ case sql :: Nil => printFragment(sql)
+ }
+ }
+
+ def printTable(name: Seq[String]): Unit = {
+ print("%s".format(name.last))
+ }
+
+ trait ExprSeq extends Seq[SortOrder]
+
+ def printExprModifiers(modifiers: (FlagSet, Seq[Seq[Any]]), pos: Position): Unit = {
+ val flagsNeedExprs = for {flag <- pickledListOrder
+ if (modifiers._1.hasFlag(flag))} yield {
+ (flag)
+ }
+
+ flagsNeedExprs.zip(modifiers._2).foreach {
+ case (SORT, Seq(order)) =>
+ if (pos == Sort) { // TODO: how to verify type of order: && order
+ // .isInstanceOf[Seq[SortOrder]]) {
+ println()
+ if (modifiers._1.hasFlag(GLOBAL)) {
+ print("ORDER BY %s"
+ .format(order.asInstanceOf[Seq[SortOrder]].map {
+ // walk around for a sparkSQL bug
+ s => {
+ s.child match {
+ case a: Alias =>
+ val qualifierPrefix = a.qualifier
+ .map(_ + ".").getOrElse("")
+ s"$qualifierPrefix${
+ quoteIdentifier(a
+ .name)
+ }"
+
+ case other => other.sql
+ }
+ } + " " + s.direction.sql + " " + s.nullOrdering.sql
+ }.mkString(", ")))
+ } else {
+ print("SORT BY %s".format(order.asInstanceOf[Seq[SortOrder]].map {
+ // walk around for a sparkSQL bug
+ s => {
+ s.child match {
+ case a: Alias =>
+ val qualifierPrefix = a.qualifier.map(_ + ".")
+ .getOrElse("")
+ s"$qualifierPrefix${ quoteIdentifier(a.name) }"
+
+ case other => other.sql
+ }
+ } + " " + s.direction.sql + " " + s.nullOrdering.sql
+ }.mkString(", ")))
+ }
+ }
+ case (LIMIT, Seq(limitExpr)) =>
+ if (pos == Limit && limitExpr.isInstanceOf[Expression]) {
+ println()
+ print(s"LIMIT ${ limitExpr.asInstanceOf[Expression].sql }")
+ }
+ case (EXPAND, Seq(_, _)) =>
+ case _ =>
+ throw new UnsupportedOperationException(s"unsupported modifiers: ${
+ flagToString(modifiers
+ ._1)
+ }")
+ }
+ }
+
+ override def printFragment(frag: Fragment): Unit = {
+ require(
+ frag.Supported,
+ "Fragment is not supported. Current frag:\n" + frag)
+
+ frag match {
+ case spjg@SPJGFragment(select, from, where, groupby, having, alias, modifiers) =>
+ if (alias.nonEmpty) {
+ print("(")
+ }
+ printSelect(select, modifiers._1)
+ if (from.nonEmpty) {
+ println()
+ }
+ printFrom(from)
+ if (where.nonEmpty) {
+ println()
+ }
+ printWhere(where)
+ if (groupby._1.nonEmpty) {
+ println()
+ }
+ printGroupby(groupby)
+ if (having.nonEmpty) {
+ println()
+ }
+ printHaving(having)
+ if (modifiers._1.hasFlag(SORT)) {
+ printExprModifiers(modifiers, Sort)
+ }
+ if (modifiers._1.hasFlag(LIMIT)) {
+ printExprModifiers(modifiers, Limit)
+ }
+ alias.map { case a: String => print(") %s ".format(a)) }
+ case union@UNIONFragment(sqls, alias, modifiers) =>
+ if (alias.nonEmpty) {
+ print("(")
+ }
+ printUnion(sqls)
+ if (modifiers._1.hasFlag(LIMIT)) {
+ printExprModifiers(modifiers, Limit)
+ }
+ alias.map { case a: String => print(") %s ".format(a)) }
+ case table@TABLEFragment(name, alias, modifiers) =>
+ printTable(name)
+ alias.map { case a: String =>
+ if (!(a == name.last)) {
+ print(" %s ".format(a))
+ }
+ }
+ case _ =>
+ throw new UnsupportedOperationException(s"unsupported plan $frag")
+ }
+ }
+ }
+
+ /**
+ * A sql fragment printer which is one single line for a fragment.
+ */
+ class SQLFragmentOneLinePrinter(out: PrintWriter) extends SQLFragmentCompactPrinter(out) {
+ protected val sep = " "
+
+ override def println() { print(sep) }
+ }
+
+ /** @group Printers */
+ protected def render(what: Any, mkPrinter: PrintWriter => FragmentPrinter): String = {
+ val buffer = new StringWriter()
+ val writer = new PrintWriter(buffer)
+ var printer = mkPrinter(writer)
+
+ printer.print(what)
+ writer.flush()
+ buffer.toString
+ }
+
+ def asCompactString(t: Fragment): String = render(t, newSQLFragmentCompactPrinter)
+
+ def asOneLineString(t: Fragment): String = render(t, newSQLFragmentOneLinePrinter)
+
+ def newSQLFragmentCompactPrinter(writer: PrintWriter): SQLFragmentCompactPrinter = {
+ new SQLFragmentCompactPrinter(
+ writer)
+ }
+
+ def newSQLFragmentCompactPrinter(stream: OutputStream): SQLFragmentCompactPrinter = {
+ newSQLFragmentCompactPrinter(
+ new PrintWriter(stream))
+ }
+
+ def newSQLFragmentOneLinePrinter(writer: PrintWriter): SQLFragmentOneLinePrinter = {
+ new SQLFragmentOneLinePrinter(
+ writer)
+ }
+
+ def newSQLFragmentOneLinePrinter(stream: OutputStream): SQLFragmentOneLinePrinter = {
+ newSQLFragmentOneLinePrinter(
+ new PrintWriter(stream))
+ }
+}
+// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuild.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuild.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuild.scala
new file mode 100644
index 0000000..8c52d28
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuild.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import java.util.concurrent.atomic.AtomicLong
+
+class SQLBuild private (
+ nextSubqueryId: AtomicLong,
+ subqueryPrefix: String) {
+ self =>
+
+ def this(subqueryPrefix: String) =
+ this(new AtomicLong(0), subqueryPrefix)
+
+ def newSubqueryName(): String = s"${subqueryPrefix}${nextSubqueryId.getAndIncrement()}"
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuildDSL.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuildDSL.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuildDSL.scala
new file mode 100644
index 0000000..307fff0
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuildDSL.scala
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, BitwiseAnd, Cast, Expression, Grouping, GroupingID, Literal, NamedExpression, ShiftRight}
+import org.apache.spark.sql.catalyst.plans.JoinType
+import org.apache.spark.sql.types.{ByteType, IntegerType}
+
+import org.apache.carbondata.mv.plans.modular
+import org.apache.carbondata.mv.plans.modular.Flags._
+import org.apache.carbondata.mv.plans.modular.ModularPlan
+
+trait SQLBuildDSL {
+
+ abstract class Fragment extends Product {
+ // this class corresponding to TreeNode of SparkSQL or Tree of Scala compiler
+ def canEqual(that: Any): Boolean = {
+ throw new UnsupportedOperationException
+ }
+
+ def productArity: Int = throw new UnsupportedOperationException
+
+ def productElement(n: Int): Any = throw new UnsupportedOperationException
+
+ // TODO: add library needed for SQL building
+ def isUnsupported: Boolean
+
+ def Supported: Boolean
+
+ // def pos: Position
+ def alias: Option[String]
+ }
+
+ case class SPJGFragment(
+ select: Seq[NamedExpression] = Nil, // TODO: more general, change to Seq[Either[Fragment,
+ // NamedExpression]]
+ from: Seq[(Fragment, Option[JoinType], Seq[Expression])] = Nil,
+ where: Seq[Expression] = Nil,
+ groupby: (Seq[Expression], Seq[Seq[Expression]]) = (Nil, Nil),
+ having: Seq[Expression] = Nil,
+ alias: Option[String] = None,
+ modifiers: (FlagSet, Seq[Seq[Any]])) extends Fragment {
+ override def isUnsupported: Boolean = {
+ select == Nil || from == Nil ||
+ from.map(_._1.isUnsupported).foldLeft(false)(_ || _)
+ }
+
+ override def Supported: Boolean = !isUnsupported
+ }
+
+ // TODO: find a scheme to break up fragmentExtract() using unapply
+ //
+ // object SPJGFragment {
+ // type ReturnType = (Seq[NamedExpression],Seq[(Fragment, Option[JoinType],Seq[Expression])
+ // ],Seq[Expression],Seq[Expression],Seq[Expression],Option[String])
+ //
+ // def unapply(plan: ModularPlan): Option[ReturnType] = fragmentExtract(plan, None)
+ // }
+
+ case class TABLEFragment(
+ table: Seq[String] = Nil,
+ alias: Option[String] = None,
+ modifiers: (FlagSet, Seq[Seq[Any]])) extends Fragment {
+ override def isUnsupported: Boolean = table == Nil
+
+ override def Supported: Boolean = !isUnsupported
+ }
+
+ case class UNIONFragment(
+ union: Seq[Fragment] = Nil,
+ alias: Option[String] = None,
+ modifiers: (FlagSet, Seq[Seq[Any]])) extends Fragment {
+ override def isUnsupported: Boolean = {
+ union == Nil ||
+ union.map(_.isUnsupported).foldLeft(false)(_ || _)
+ }
+
+ override def Supported: Boolean = !isUnsupported
+ }
+
+ val UnsupportedFragment: Fragment = new Fragment {
+ def isUnsupported = true
+
+ def Supported = false
+
+ def alias = None
+ }
+
+ /**
+ * Turns a bunch of string segments into a single string and separate each segment by a space.
+ * The segments are trimmed so only a single space appears in the separation.
+ * For example, `build("a", " b ", " c")` becomes "a b c".
+ */
+ protected def build(segments: String*): String = {
+ segments.map(_.trim).filter(_.nonEmpty).mkString(" ")
+ }
+
+ def fragmentExtract(plan: ModularPlan, alias: Option[String]): Fragment = {
+ if (plan.rewritten) {
+ fragmentExtract_Rewritten(plan, alias)
+ } else {
+ fragmentExtract_NonRewritten(plan, alias)
+ }
+ }
+
+ // Rewritten portion of query plan
+ private def fragmentExtract_Rewritten(
+ plan: ModularPlan,
+ alias: Option[String]): Fragment = {
+ plan match {
+ case s1@modular.Select(_, _, _, _, _,
+ Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if (!s1.skip && !g.skip && !s2.skip) =>
+ extractRewrittenOrNonRewrittenSelectGroupBySelect(s1, g, s2, alias)
+
+ case s1@modular.Select(_, _, _, _, _,
+ Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if (s1.skip && g.skip && s2.skip) =>
+ extractRewrittenOrNonRewrittenSelectGroupBySelect(s1, g, s2, alias)
+
+ case s1@modular.Select(_, _, _, _, _, Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if (!s1.skip && !g.skip && s2.skip) =>
+ extractRewrittenSelectGroupBy(s1, g, alias)
+
+ case s1@modular.Select(_, _, _, _, _, Seq(s2@modular.Select(_, _, _, _, _, _, _, _, _, _)),
+ _, _, _, _) if (!s1.skip && s2.skip) =>
+ extractRewrittenSelect(s1, alias)
+
+ case other => extractSimpleOperator(other, alias)
+ }
+ }
+
+ // Non-rewritten portion of query plan
+ private def fragmentExtract_NonRewritten(
+ plan: ModularPlan,
+ alias: Option[String]): Fragment = {
+ plan match {
+ case s1@modular.Select(_, _, _, _, _,
+ Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if (s1.aliasMap.isEmpty && !g.rewritten) =>
+ extractRewrittenOrNonRewrittenSelectGroupBySelect(s1, g, s2, alias)
+
+ case g@modular.GroupBy(_, _, _, _, s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)
+ if (g.alias.isEmpty && !s2.rewritten) =>
+ val fragmentList = s2.children.zipWithIndex
+ .map { case (child, index) => fragmentExtract(child, s2.aliasMap.get(index)) }
+ val fList = s2.joinEdges.map {
+ e => {
+ (e.right, (fragmentList(e.right), Some(e.joinType), s2
+ .extractRightEvaluableConditions(s2.children(e.left), s2.children(e.right))))
+ }
+ }.toMap
+ val from = (0 to fragmentList.length - 1)
+ .map(index => fList.get(index).getOrElse((fragmentList(index), None, Nil)))
+ val excludesPredicate = from.flatMap(_._3).toSet
+ val (select, (groupByExprs, groupingSet)) = addGroupingSetIfNeeded(g, s2)
+ SPJGFragment(
+ select,
+ from,
+ s2.predicateList.filter { p => !excludesPredicate(p) },
+ (groupByExprs, groupingSet),
+ Nil,
+ alias,
+ (g.flags, g.flagSpec))
+
+ case g@modular.GroupBy(_, _, _, _, _, _, _, _) if (g.alias.nonEmpty) =>
+ val from = Seq((fragmentExtract(g.child, g.alias), None, Nil))
+ SPJGFragment(
+ g.outputList,
+ from,
+ Nil,
+ (g.predicateList, Seq.empty),
+ Nil,
+ alias,
+ (g.flags, g.flagSpec))
+
+ case other => extractSimpleOperator(other, alias)
+ }
+ }
+
+ // used in both rewritten and non-rewritten cases
+ // currently in rewritten cases we don't consider grouping set
+ private def extractRewrittenOrNonRewrittenSelectGroupBySelect(
+ s1: modular.Select,
+ g: modular.GroupBy,
+ s2: modular.Select,
+ alias: Option[String]): Fragment = {
+ val fragmentList = s2.children.zipWithIndex
+ .map { case (child, index) => fragmentExtract(child, s2.aliasMap.get(index)) }
+ val fList = s2.joinEdges.map {
+ e => {
+ (e.right, (fragmentList(e.right), Some(e.joinType), s2
+ .extractRightEvaluableConditions(s2.children(e.left), s2.children(e.right))))
+ }
+ }.toMap
+ val from = (0 to fragmentList.length - 1)
+ .map(index => fList.get(index).getOrElse((fragmentList(index), None, Nil)))
+ val excludesPredicate = from.flatMap(_._3).toSet
+ val aliasMap = AttributeMap(g.outputList.collect { case a: Alias => (a.toAttribute, a) })
+ val windowExprs = s1.windowSpec
+ .map { case Seq(expr) => expr.asInstanceOf[Seq[NamedExpression]] }
+ .foldLeft(Seq.empty.asInstanceOf[Seq[NamedExpression]])(_ ++ _)
+ val having = s1.predicateList
+ .map { case attr: Attribute => aliasMap.get(attr).map(_.child)
+ .getOrElse(attr);
+ case expr: Expression => expr.transform { case a: Alias => a.child };
+ case other => other
+ }
+
+ val (select_g, (groupByExprs, groupingSet)) = addGroupingSetIfNeeded(g, s2)
+
+ val gSet = AttributeSet(g.outputList.map(_.toAttribute))
+ val sSet = AttributeSet(s1.outputList.map(_.toAttribute))
+ val select = if (groupingSet.nonEmpty) {
+ if (gSet.equals(sSet) && windowExprs.isEmpty) {
+ select_g
+ } else {
+ throw new UnsupportedOperationException
+ }
+ } else
+ // TODO: how to handle alias of attribute in MV
+ {
+ s1.outputList.map { attr => aliasMap.get(attr.toAttribute).getOrElse(attr) } ++ windowExprs
+ }
+
+ SPJGFragment(
+ select, // select
+ from, // from
+ s2.predicateList.filter { p => !excludesPredicate(p) }, // where
+ (groupByExprs, groupingSet), // group by
+ having, // having
+ alias,
+ (s1.flags, s1.flagSpec))
+ }
+
+ // used in rewritten cases only -- don't consider grouping set
+ private def extractRewrittenSelectGroupBy(
+ s1: modular.Select,
+ g: modular.GroupBy,
+ alias: Option[String]): Fragment = {
+ val fragment = fragmentExtract(g.child, g.alias)
+ val from = Seq((fragment, None, Nil))
+ val aliasMap = AttributeMap(g.outputList.collect { case a: Alias => (a.toAttribute, a) })
+ val windowExprs = s1.windowSpec
+ .map { case Seq(expr) => expr.asInstanceOf[Seq[NamedExpression]] }
+ .foldLeft(Seq.empty.asInstanceOf[Seq[NamedExpression]])(_ ++ _)
+ val having = s1.predicateList
+ .map { case attr: Attribute => aliasMap.get(attr).map(_.child)
+ .getOrElse(attr);
+ case expr: Expression => expr.transform { case a: Alias => a.child };
+ case other => other
+ }
+
+ val (select_g, (groupByExprs, groupingSet)) = (g.outputList, (g.predicateList, Seq.empty))
+
+ // TODO: how to handle alias of attribute in MV
+ val select = s1.outputList.map { attr => aliasMap.get(attr.toAttribute).getOrElse(attr) } ++
+ windowExprs
+ SPJGFragment(
+ select, // select
+ from, // from
+ Nil, // where
+ (groupByExprs, groupingSet), // group by
+ having, // having
+ alias,
+ (s1.flags, s1.flagSpec))
+ }
+
+ private def extractRewrittenSelect(s1: modular.Select, alias: Option[String]): Fragment = {
+ val fragment = fragmentExtract(s1.children(0), s1.aliasMap.get(0))
+ val from = Seq((fragment, None, Nil))
+ val windowExprs = s1.windowSpec
+ .map { case Seq(expr) => expr.asInstanceOf[Seq[NamedExpression]] }
+ .foldLeft(Seq.empty.asInstanceOf[Seq[NamedExpression]])(_ ++ _)
+ val select = s1.outputList.map(_.toAttribute)
+
+ SPJGFragment(
+ select, // select
+ from, // from
+ s1.predicateList, // where
+ (Nil, Nil), // group by
+ Nil, // having
+ alias,
+ (s1.flags, s1.flagSpec))
+ }
+
+ private def extractSimpleOperator(
+ operator: ModularPlan,
+ alias: Option[String]): Fragment = {
+ operator match {
+ case s@modular.Select(_, _, _, _, _, _, _, _, _, _) =>
+ val fragmentList = s.children.zipWithIndex
+ .map { case (child, index) => fragmentExtract(child, s.aliasMap.get(index)) }
+ val fList = s.joinEdges.map {
+ e => {
+ (e.right, (fragmentList(e.right), Some(e.joinType), s
+ .extractRightEvaluableConditions(s.children(e.left), s.children(e.right))))
+ }
+ }.toMap
+ val from = (0 to fragmentList.length - 1)
+ .map(index => fList.get(index).getOrElse((fragmentList(index), None, Nil)))
+ val excludesPredicate = from.flatMap(_._3).toSet
+ val windowExprs = s.windowSpec
+ .map { case Seq(expr) => expr.asInstanceOf[Seq[NamedExpression]] }
+ .foldLeft(Seq.empty.asInstanceOf[Seq[NamedExpression]])(_ ++ _)
+ val select = s.outputList ++ windowExprs
+
+ SPJGFragment(
+ select, // select
+ from, // from
+ s.predicateList.filter { p => !excludesPredicate(p) }, // where
+ (Nil, Nil), // group by
+ Nil, // having
+ alias,
+ (s.flags, s.flagSpec))
+
+ case u@modular.Union(_, _, _) =>
+ UNIONFragment(
+ u.children.zipWithIndex.map { case (child, index) => fragmentExtract(child, None) },
+ alias,
+ (u.flags, u.flagSpec))
+
+ case d@modular.ModularRelation(_, _, _, _, _) =>
+ if (d.databaseName != null && d.tableName != null) {
+ TABLEFragment(
+ Seq(d.databaseName, d.tableName), alias, (d.flags, d.rest))
+ } else {
+ TABLEFragment(Seq((d.output).toString()), alias, (d.flags, d.rest))
+ }
+
+ case h@modular.HarmonizedRelation(_) =>
+ fragmentExtract(h.source, alias)
+
+ case _ => UnsupportedFragment
+ }
+ }
+
+ private def addGroupingSetIfNeeded(g: modular.GroupBy, s: modular.Select) = {
+ if (g.flags.hasFlag(EXPAND)) {
+ assert(g.predicateList.length > 1)
+ val flagsNeedExprs =
+ for {flag <- pickledListOrder if (g.flags.hasFlag(flag))} yield {
+ flag
+ }
+ flagsNeedExprs.zip(g.flagSpec).collect {
+ case (EXPAND, Seq(projections_, output_, numOriginalOutput_)) =>
+ val output = output_.asInstanceOf[Seq[Attribute]]
+ val projections = projections_.asInstanceOf[Seq[Seq[Expression]]]
+ val numOriginalOutput = numOriginalOutput_.asInstanceOf[Int]
+
+ // The last column of Expand is always grouping ID
+ val gid = output.last
+
+ val groupByAttributes = g.predicateList.dropRight(1).map(_.asInstanceOf[Attribute])
+ // Assumption: project's projectList is composed of
+ // 1) the original output (Project's child.output),
+ // 2) the aliased group by expressions.
+ val expandedAttributes = s.output.drop(numOriginalOutput)
+ val groupByExprs = s.outputList.drop(numOriginalOutput).map(_.asInstanceOf[Alias].child)
+
+ // a map from group by attributes to the original group by expressions.
+ val groupByAttrMap = AttributeMap(groupByAttributes.zip(groupByExprs))
+ // a map from expanded attributes to the original group by expressions.
+ val expandedAttrMap = AttributeMap(expandedAttributes.zip(groupByExprs))
+
+ val groupingSet: Seq[Seq[Expression]] = projections.map { project =>
+ // Assumption: expand.projections is composed of
+ // 1) the original output (Project's child.output),
+ // 2) expanded attributes(or null literal)
+ // 3) gid, which is always the last one in each project in Expand
+ project.drop(numOriginalOutput).dropRight(1).collect {
+ case attr: Attribute if expandedAttrMap.contains(attr) => expandedAttrMap(attr)
+ }
+ }
+
+ val aggExprs = g.outputList.map { case aggExpr =>
+ val originalAggExpr = aggExpr.transformDown {
+ // grouping_id() is converted to VirtualColumn.groupingIdName by Analyzer. Revert
+ // it back.
+ case ar: AttributeReference if ar == gid => GroupingID(Nil)
+ case ar: AttributeReference if groupByAttrMap.contains(ar) => groupByAttrMap(ar)
+ case a@Cast(
+ BitwiseAnd(
+ ShiftRight(ar: AttributeReference, Literal(value: Any, IntegerType)),
+ Literal(1, IntegerType)), ByteType, None) if ar == gid =>
+ // for converting an expression to its original SQL format grouping(col)
+ val idx = groupByExprs.length - 1 - value.asInstanceOf[Int]
+ groupByExprs.lift(idx).map(Grouping).getOrElse(a)
+ }
+
+ originalAggExpr match {
+ // Ancestor operators may reference the output of this grouping set,
+ // and we use exprId to generate a unique name for each attribute, so we should
+ // make sure the transformed aggregate expression won't change the output,
+ // i.e. exprId and alias name should remain the same.
+ case ne: NamedExpression if ne.exprId == aggExpr.exprId => ne
+ case e => Alias(e, aggExpr.name)(exprId = aggExpr.exprId)
+ }
+ }
+ (aggExprs, (groupByExprs, groupingSet))
+
+ }.head
+ } else {
+ (g.outputList, (g.predicateList, Seq.empty))
+ }
+ }
+}
+
+object SQLBuildDSL extends SQLBuildDSL
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
new file mode 100644
index 0000000..4bc8b97
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import java.util.concurrent.atomic.AtomicLong
+
+import scala.collection.immutable
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeMap, AttributeReference, AttributeSet, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
+
+import org.apache.carbondata.mv.expressions.modular._
+import org.apache.carbondata.mv.plans._
+import org.apache.carbondata.mv.plans.modular.{Select, _}
+import org.apache.carbondata.mv.plans.util.SQLBuildDSL.Fragment
+
+// TODO: generalize this using SQLBuild to handle nested sub-query expression, using the
+// correspondence
+// SQLBuild <--> QueryRewrite
+class SQLBuilder private(
+ modularPlan: ModularPlan,
+ nextSubqueryId: AtomicLong,
+ subqueryPrefix: String) {
+
+ def this(modularPlan: ModularPlan) = {
+ this(
+ SQLBuilder.subqueryPreprocess(modularPlan, new AtomicLong()),
+ new AtomicLong(0),
+ "gen_subquery_")
+ }
+
+ // only allow one level scalar subquery
+ def this(modularPlan: ModularPlan, subqueryPrefix: String) = {
+ this(SQLBuilder.screenOutPlanWithSubquery(modularPlan), new AtomicLong(0), subqueryPrefix)
+ }
+
+ private def newSubqueryName(): String = {
+ s"${ subqueryPrefix }${
+ nextSubqueryId.getAndIncrement()
+ }"
+ }
+
+ def fragmentExtract: Fragment = {
+ val finalPlan: ModularPlan = SQLizer.execute(modularPlan)
+ SQLBuildDSL.fragmentExtract(finalPlan, None)
+ }
+
+ object SQLizer extends RuleExecutor[ModularPlan] {
+ override protected def batches: Seq[Batch] = {
+ Seq(
+ Batch(
+ "Recover Scoping Info", Once,
+ // Clean up qualifiers due to erasure of sub-query aliases to use base tables as
+ // qualifiers
+ // This complements AddSubquery to handle corner case qMV3 in Tpcds_1_4_Querybatch.scala
+ CleanupQualifier,
+ // Insert sub queries on top of operators that need to appear after FROM clause.
+ AddSubquery
+ )
+ )
+ }
+
+ object CleanupQualifier extends Rule[ModularPlan] {
+ override def apply(tree: ModularPlan): ModularPlan = {
+ tree transformDown {
+ case s@modular.Select(_, _, _, _, _,
+ Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if !s.rewritten && s2.children.forall { _.isInstanceOf[modular.LeafNode] } =>
+ val attrMap = AttributeMap(s2.outputList
+ .collect { case a: Alias if a.child.isInstanceOf[Attribute] => (a.toAttribute, a) })
+ cleanupQualifier(s, s2.aliasMap, s2.children, attrMap)
+
+ case g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)
+ if !g.rewritten && s2.children.forall { _.isInstanceOf[modular.LeafNode] } =>
+ val attrMap = AttributeMap(s2.outputList
+ .collect { case a: Alias if a.child.isInstanceOf[Attribute] => (a.toAttribute, a) })
+ cleanupQualifier(g, s2.aliasMap, s2.children, attrMap)
+
+ case s@modular.Select(_, _, _, _, _, _, _, _, _, _)
+ if !s.rewritten && s.children.forall { _.isInstanceOf[modular.LeafNode] } =>
+ cleanupQualifier(s, s.aliasMap, s.children, AttributeMap(Seq.empty[(Attribute, Alias)]))
+
+ }
+ }
+
+ private def cleanupQualifier(node: ModularPlan,
+ aliasMap: Map[Int, String],
+ children: Seq[ModularPlan],
+ attrMap: AttributeMap[Alias]): ModularPlan = {
+ node transform {
+ case plan if !plan.rewritten && !plan.isInstanceOf[modular.LeafNode] =>
+ plan.transformExpressions {
+ case ref: Attribute =>
+ val i = children.indexWhere(_.outputSet.contains(ref))
+ if (i > -1) {
+ // this is a walk around for mystery of spark qualifier
+ if (aliasMap.nonEmpty && aliasMap(i).nonEmpty) {
+ AttributeReference(
+ ref.name,
+ ref.dataType)(exprId = ref.exprId, qualifier = Option(aliasMap(i)))
+ } else {
+ ref
+ }
+ } else {
+ attrMap.get(ref) match {
+ case Some(alias) => Alias(alias.child, alias.name)(exprId = alias.exprId)
+ case None => ref
+ }
+ }
+ }
+ }
+ }
+ }
+
+ object AddSubquery extends Rule[ModularPlan] {
+ override def apply(tree: ModularPlan): ModularPlan = {
+ tree transformUp {
+ case s@modular.Select(_, _, _, _, _,
+ Seq(g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)), _, _, _, _)
+ if s2.children.forall { _.isInstanceOf[modular.LeafNode] } => s
+
+ case g@modular.GroupBy(_, _, _, _,
+ s2@modular.Select(_, _, _, _, _, _, _, _, _, _), _, _, _)
+ if s2.children.forall { _.isInstanceOf[modular.LeafNode] } => g
+
+ case s@modular.Select(_, _, _, _, _, _, _, _, _, _)
+ if !s.rewritten && !s.children.forall { _.isInstanceOf[modular.LeafNode] } =>
+ var list: List[(Int, String)] = List()
+ var newS = s.copy()
+ s.children.zipWithIndex.filterNot { _._1.isInstanceOf[modular.LeafNode] }.foreach {
+ case (child: ModularPlan, index) if (!s.aliasMap.contains(index)) =>
+ val subqueryName = newSubqueryName()
+ val windowAttributeSet = if (child.isInstanceOf[Select]) {
+ val windowExprs = child.asInstanceOf[Select].windowSpec
+ .map { case Seq(expr) => expr.asInstanceOf[Seq[NamedExpression]] }
+ .foldLeft(Seq.empty.asInstanceOf[Seq[NamedExpression]])(_ ++ _)
+ SQLBuilder.collectAttributeSet(windowExprs)
+ } else {
+ AttributeSet.empty
+ }
+ val subqueryAttributeSet = child.outputSet ++ windowAttributeSet
+ // TODO: how to use alias to avoid duplicate names with distinct
+ // exprIds
+ // val duplicateNames = collectDuplicateNames(subqueryAttributeSet)
+ // .toSet
+ if (SQLBuilder.collectDuplicateNames(subqueryAttributeSet).nonEmpty) {
+ new UnsupportedOperationException(s"duplicate name(s): ${
+ child.output
+ .map(_.toString + ", ")
+ }")
+ }
+ list = list :+ ((index, subqueryName))
+ newS = newS.transformExpressions {
+ case ref: Attribute if (subqueryAttributeSet.contains(ref)) =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(subqueryName))
+ case alias: Alias if (subqueryAttributeSet.contains(alias.toAttribute)) =>
+ Alias(alias.child, alias.name)(
+ exprId = alias.exprId,
+ qualifier = Some(subqueryName))
+ }
+
+ case _ =>
+ }
+ if (list.nonEmpty) {
+ list = list ++ s.aliasMap.toSeq
+ newS.copy(aliasMap = list.toMap)
+ } else {
+ newS
+ }
+
+ case g@modular.GroupBy(_, _, _, _, _, _, _, _) if (!g.rewritten && g.alias.isEmpty) =>
+ val newG = if (g.outputList.isEmpty) {
+ val ol = g.predicateList.map { case a: Attribute => a }
+ g.copy(outputList = ol)
+ } else {
+ g
+ }
+ val subqueryName = newSubqueryName()
+ val subqueryAttributeSet = newG.child.outputSet
+ if (SQLBuilder.collectDuplicateNames(subqueryAttributeSet).nonEmpty) {
+ new UnsupportedOperationException(s"duplicate name(s): ${
+ newG.child.output.map(_.toString + ", ")
+ }")
+ }
+ newG.transformExpressions {
+ case ref: AttributeReference if (subqueryAttributeSet.contains(ref)) =>
+ AttributeReference(ref.name, ref.dataType)(
+ exprId = ref.exprId,
+ qualifier = Some(subqueryName))
+ case alias: Alias if (subqueryAttributeSet.contains(alias.toAttribute)) =>
+ Alias(alias.child, alias.name)(
+ exprId = alias.exprId,
+ qualifier = Some(subqueryName))
+ }.copy(alias = Some(subqueryName))
+ }
+ }
+ }
+ }
+
+}
+
+object SQLBuilder {
+ def collectAttributeSet(outputList: Seq[Expression]): AttributeSet = {
+ AttributeSet(outputList
+ .collect {
+ case a@Alias(_, _) => a.toAttribute
+ case a: Attribute => a
+ })
+ }
+
+ def collectDuplicateNames(s: AttributeSet): immutable.Iterable[String] = {
+ s.baseSet.map(_.a).groupBy(_.name).collect { case (x, ys) if ys.size > 1 => x }
+ }
+
+ def subqueryPreprocess(plan: ModularPlan, nextScalarSubqueryId: AtomicLong): ModularPlan = {
+ def newSubqueryName(nextScalarSubqueryId: AtomicLong): String = {
+ s"gen_expression_${
+ nextScalarSubqueryId
+ .getAndIncrement()
+ }_"
+ }
+
+ plan.transformAllExpressions {
+ case s: ModularSubquery =>
+ if (s.children.isEmpty) {
+ val subqueryName = newSubqueryName(nextScalarSubqueryId)
+ SubqueryHolder(s"(${ s.plan.asOneLineSQL(subqueryName) })")
+ } else {
+ throw new UnsupportedOperationException(s"Expression $s can't converted to SQL")
+ }
+ case o => o
+ }
+ }
+
+ def screenOutPlanWithSubquery(plan: ModularPlan): ModularPlan = {
+ plan.transformAllExpressions {
+ case s: ModularSubquery =>
+ throw new UnsupportedOperationException(s"Nested scala subquery $s doesn't supported")
+ case e => e
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Signature.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Signature.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Signature.scala
new file mode 100644
index 0000000..c46124b
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/Signature.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.trees.TreeNode
+
+case class Signature(groupby: Boolean = true, datasets: Set[String] = Set.empty)
+
+abstract class SignatureRule[BaseType <: TreeNode[BaseType]] extends Logging {
+ def apply(subplan: BaseType, signatures: Seq[Option[Signature]]): Option[Signature]
+}
+
+abstract class SignatureGenerator[BaseType <: TreeNode[BaseType]] extends Logging {
+ protected val rule: SignatureRule[BaseType]
+
+ def generate(subplan: BaseType): Option[Signature] = {
+ generateUp(subplan)
+ }
+
+ protected def generateChildren(subplan: BaseType, nextOperation: BaseType => Option[Signature]
+ ): Seq[Option[Signature]] = {
+ subplan.children.map { child =>
+ nextOperation(child.asInstanceOf[BaseType])
+ }
+ }
+
+ def generateUp(subplan: BaseType): Option[Signature] = {
+ val childSignatures = generateChildren(subplan, t => generateUp(t))
+ val lastSignature = rule(subplan, childSignatures)
+ lastSignature
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/TableCluster.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/TableCluster.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/TableCluster.scala
new file mode 100644
index 0000000..021518c
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/TableCluster.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.plans.util
+
+import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty, JsonRawValue}
+import com.google.common.base.Objects
+
+class TableCluster @JsonCreator()(@JsonProperty("fact") @JsonRawValue fact: Set[String],
+ @JsonProperty("dimension") @JsonRawValue dimension: Set[String]) {
+
+ // @JsonProperty
+ def getFact(): Set[String] = {
+ fact
+ }
+
+ //
+ // @JsonProperty
+ def getDimension(): Set[String] = {
+ dimension
+ }
+
+ @Override
+ override def toString: String = {
+ Objects.toStringHelper(this)
+ .add("fact", fact)
+ .add("dimension", dimension)
+ .toString
+ }
+
+ /*
+ @Override
+ def toString = {
+ MoreObjects.toStringHelper(this)
+ .add("fact", fact)
+ .add("dimension", dimension)
+ .toString
+ }
+ *
+ */
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/ModularPlanTest.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/ModularPlanTest.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/ModularPlanTest.scala
new file mode 100644
index 0000000..0c68a1f
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/ModularPlanTest.scala
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+import java.io.File
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.test.util.{PlanTest, QueryTest}
+
+import org.apache.carbondata.mv.plans.modular
+import org.apache.carbondata.mv.plans.modular.{ModularPlan, OneRowTable, Select}
+import org.apache.carbondata.mv.plans.modular.Flags._
+
+/**
+ * Provides helper methods for comparing plans.
+ */
+abstract class ModularPlanTest extends QueryTest with PredicateHelper {
+
+ /**
+ * Since attribute references are given globally unique ids during analysis,
+ * we must normalize them to check if two different queries are identical.
+ */
+ protected def normalizeExprIds(plan: ModularPlan): plan.type = {
+ plan transformAllExpressions {
+ case s: ScalarSubquery =>
+ s.copy(exprId = ExprId(0))
+ case e: Exists =>
+ e.copy(exprId = ExprId(0))
+ case l: ListQuery =>
+ l.copy(exprId = ExprId(0))
+ case a: AttributeReference =>
+ AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0))
+ case a: Alias =>
+ Alias(a.child, a.name)(exprId = ExprId(0))
+ case ae: AggregateExpression =>
+ ae.copy(resultId = ExprId(0))
+ }
+ }
+
+ /**
+ * Normalizes plans:
+ * - Filter the filter conditions that appear in a plan. For instance,
+ * ((expr 1 && expr 2) && expr 3), (expr 1 && expr 2 && expr 3), (expr 3 && (expr 1 && expr 2)
+ * etc., will all now be equivalent.
+ * - Sample the seed will replaced by 0L.
+ * - Join conditions will be resorted by hashCode.
+ */
+ protected def normalizePlan(plan: LogicalPlan): LogicalPlan = {
+ plan transform {
+ case filter@Filter(condition: Expression, child: LogicalPlan) =>
+ Filter(
+ splitConjunctivePredicates(condition).map(rewriteEqual(_)).sortBy(_.hashCode())
+ .reduce(And), child)
+ case sample: Sample =>
+ sample.copy(seed = 0L)(true)
+ case join@Join(left, right, joinType, condition) if condition.isDefined =>
+ val newCondition =
+ splitConjunctivePredicates(condition.get).map(rewriteEqual(_)).sortBy(_.hashCode())
+ .reduce(And)
+ Join(left, right, joinType, Some(newCondition))
+ }
+ }
+
+ /**
+ * Rewrite [[EqualTo]] and [[EqualNullSafe]] operator to keep order. The following cases will be
+ * equivalent:
+ * 1. (a = b), (b = a);
+ * 2. (a <=> b), (b <=> a).
+ */
+ private def rewriteEqual(condition: Expression): Expression = {
+ condition match {
+ case eq@EqualTo(l: Expression, r: Expression) =>
+ Seq(l, r).sortBy(_.hashCode()).reduce(EqualTo)
+ case eq@EqualNullSafe(l: Expression, r: Expression) =>
+ Seq(l, r).sortBy(_.hashCode()).reduce(EqualNullSafe)
+ case _ => condition // Don't reorder.
+ }
+ }
+
+ //
+ // /** Fails the test if the two plans do not match */
+ // protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) {
+ // val normalized1 = normalizePlan(normalizeExprIds(plan1))
+ // val normalized2 = normalizePlan(normalizeExprIds(plan2))
+ // if (normalized1 != normalized2) {
+ // fail(
+ // s"""
+ // |== FAIL: Plans do not match ===
+ // |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")}
+ // """.stripMargin)
+ // }
+ // }
+ //
+ // /** Fails the test if the two expressions do not match */
+ // protected def compareExpressions(e1: Expression, e2: Expression): Unit = {
+ // comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation))
+ // }
+ //
+ // /** Fails the test if the join order in the two plans do not match */
+ // protected def compareJoinOrder(plan1: LogicalPlan, plan2: LogicalPlan) {
+ // val normalized1 = normalizePlan(normalizeExprIds(plan1))
+ // val normalized2 = normalizePlan(normalizeExprIds(plan2))
+ // if (!sameJoinPlan(normalized1, normalized2)) {
+ // fail(
+ // s"""
+ // |== FAIL: Plans do not match ===
+ // |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")}
+ // """.stripMargin)
+ // }
+ // }
+ //
+ // /** Consider symmetry for joins when comparing plans. */
+ // private def sameJoinPlan(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ // (plan1, plan2) match {
+ // case (j1: Join, j2: Join) =>
+ // (sameJoinPlan(j1.left, j2.left) && sameJoinPlan(j1.right, j2.right)) ||
+ // (sameJoinPlan(j1.left, j2.right) && sameJoinPlan(j1.right, j2.left))
+ // case (p1: Project, p2: Project) =>
+ // p1.projectList == p2.projectList && sameJoinPlan(p1.child, p2.child)
+ // case _ =>
+ // plan1 == plan2
+ // }
+ // }
+ /** Fails the test if the corresponding pairs of plans do not match */
+ protected def comparePlanCollections(planSet1: Seq[String], planSet2: Seq[String]) {
+ for ((plan1, plan2) <- planSet1 zip planSet2) {
+ compareMessages(plan1, plan2)
+ }
+ }
+
+ /** Fails the test if the two plans do not match */
+ /** Only expressionIds are normalized. This is enough for our test cases */
+ /** For more general normalization, see Spark PlanTest.scala for Logical Plan */
+ protected def comparePlans(plan1: ModularPlan, plan2: ModularPlan) {
+ val normalized1 = normalizeExprIds(plan1)
+ val normalized2 = normalizeExprIds(plan2)
+ if (normalized1 != normalized2) {
+ fail(
+ s"""
+ |== FAIL: Plans do not match ===
+ |${ sideBySide(normalized1.treeString, normalized1.treeString).mkString("\n") }
+ """.stripMargin)
+ }
+ }
+
+ /** Fails the test if the two expressions do not match */
+ protected def compareExpressions(e1: Seq[Expression], e2: Seq[Expression]): Unit = {
+ comparePlans(
+ Select(Nil, Nil, e1, Map.empty, Nil, Seq(OneRowTable), NoFlags, Seq.empty, Seq.empty), modular
+ .Select(Nil, Nil, e2, Map.empty, Nil, Seq(OneRowTable), NoFlags, Seq.empty, Seq.empty))
+ }
+
+ protected def compareMessages(msg1: String, msg2: String) {
+ if (msg1 != msg2) {
+ fail(
+ s"""
+ |== FAIL: Messages do not match ==
+ |${ sideBySide(msg1, msg2).mkString("\n") }
+ """.stripMargin)
+ }
+ }
+}
[2/4] carbondata git commit: [CARBONDATA-2474] Support Modular Plan
for Materialized View DataMap
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ffddba70/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_QueryBatch.scala
----------------------------------------------------------------------
diff --git a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_QueryBatch.scala b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_QueryBatch.scala
new file mode 100644
index 0000000..8262dfa
--- /dev/null
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_QueryBatch.scala
@@ -0,0 +1,4293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+object Tpcds_1_4_QueryBatch {
+
+ // should be random generated based on scale
+ // RC=ulist(random(1, rowcount("store_sales")/5,uniform),5);
+ val rc = Array(1000000, 1000000, 1000000, 1000000, 1000000)
+
+ // Queries the TPCDS 1.4 queries using the qualifcations values in the templates.
+
+ val tpcds1_4Queries = Seq(
+ ("q1",
+ """
+ | WITH customer_total_return AS
+ | (SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk,
+ | sum(sr_return_amt) AS ctr_total_return
+ | FROM store_returns, date_dim
+ | WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000
+ | GROUP BY sr_customer_sk, sr_store_sk)
+ | SELECT c_customer_id
+ | FROM customer_total_return ctr1, store, customer
+ | WHERE ctr1.ctr_total_return >
+ | (SELECT avg(ctr_total_return)*1.2
+ | FROM customer_total_return ctr2
+ | WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk)
+ | AND s_store_sk = ctr1.ctr_store_sk
+ | AND s_state = 'TN'
+ | AND ctr1.ctr_customer_sk = c_customer_sk
+ | ORDER BY c_customer_id LIMIT 100
+ """.stripMargin),
+ ("q2",
+ """
+ | WITH wscs as
+ | (SELECT sold_date_sk, sales_price
+ | FROM (SELECT ws_sold_date_sk sold_date_sk, ws_ext_sales_price sales_price
+ | FROM web_sales) x
+ | UNION ALL
+ | (SELECT cs_sold_date_sk sold_date_sk, cs_ext_sales_price sales_price
+ | FROM catalog_sales)),
+ | wswscs AS
+ | (SELECT d_week_seq,
+ | sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales,
+ | sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales,
+ | sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales,
+ | sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales,
+ | sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales,
+ | sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales,
+ | sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales
+ | FROM wscs, date_dim
+ | WHERE d_date_sk = sold_date_sk
+ | GROUP BY d_week_seq)
+ | SELECT d_week_seq1
+ | ,round(sun_sales1/sun_sales2,2)
+ | ,round(mon_sales1/mon_sales2,2)
+ | ,round(tue_sales1/tue_sales2,2)
+ | ,round(wed_sales1/wed_sales2,2)
+ | ,round(thu_sales1/thu_sales2,2)
+ | ,round(fri_sales1/fri_sales2,2)
+ | ,round(sat_sales1/sat_sales2,2)
+ | FROM
+ | (SELECT wswscs.d_week_seq d_week_seq1
+ | ,sun_sales sun_sales1
+ | ,mon_sales mon_sales1
+ | ,tue_sales tue_sales1
+ | ,wed_sales wed_sales1
+ | ,thu_sales thu_sales1
+ | ,fri_sales fri_sales1
+ | ,sat_sales sat_sales1
+ | FROM wswscs,date_dim
+ | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001) y,
+ | (SELECT wswscs.d_week_seq d_week_seq2
+ | ,sun_sales sun_sales2
+ | ,mon_sales mon_sales2
+ | ,tue_sales tue_sales2
+ | ,wed_sales wed_sales2
+ | ,thu_sales thu_sales2
+ | ,fri_sales fri_sales2
+ | ,sat_sales sat_sales2
+ | FROM wswscs, date_dim
+ | WHERE date_dim.d_week_seq = wswscs.d_week_seq AND d_year = 2001 + 1) z
+ | WHERE d_week_seq1=d_week_seq2-53
+ | ORDER BY d_week_seq1
+ """.stripMargin),
+ ("q3",
+ """
+ | SELECT dt.d_year, item.i_brand_id brand_id, item.i_brand brand,SUM(ss_ext_sales_price)
+ | sum_agg
+ | FROM date_dim dt, store_sales, item
+ | WHERE dt.d_date_sk = store_sales.ss_sold_date_sk
+ | AND store_sales.ss_item_sk = item.i_item_sk
+ | AND item.i_manufact_id = 128
+ | AND dt.d_moy=11
+ | GROUP BY dt.d_year, item.i_brand, item.i_brand_id
+ | ORDER BY dt.d_year, sum_agg desc, brand_id
+ | LIMIT 100
+ """.stripMargin),
+ ("q4",
+ """
+ |WITH year_total AS (
+ | SELECT c_customer_id customer_id,
+ | c_first_name customer_first_name,
+ | c_last_name customer_last_name,
+ | c_preferred_cust_flag customer_preferred_cust_flag,
+ | c_birth_country customer_birth_country,
+ | c_login customer_login,
+ | c_email_address customer_email_address,
+ | d_year dyear,
+ | sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)
+ | +ss_ext_sales_price)/2) year_total,
+ | 's' sale_type
+ | FROM customer, store_sales, date_dim
+ | WHERE c_customer_sk = ss_customer_sk AND ss_sold_date_sk = d_date_sk
+ | GROUP BY c_customer_id,
+ | c_first_name,
+ | c_last_name,
+ | c_preferred_cust_flag,
+ | c_birth_country,
+ | c_login,
+ | c_email_address,
+ | d_year
+ | UNION ALL
+ | SELECT c_customer_id customer_id,
+ | c_first_name customer_first_name,
+ | c_last_name customer_last_name,
+ | c_preferred_cust_flag customer_preferred_cust_flag,
+ | c_birth_country customer_birth_country,
+ | c_login customer_login,
+ | c_email_address customer_email_address,
+ | d_year dyear,
+ | sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)
+ | +cs_ext_sales_price)/2) ) year_total,
+ | 'c' sale_type
+ | FROM customer, catalog_sales, date_dim
+ | WHERE c_customer_sk = cs_bill_customer_sk AND cs_sold_date_sk = d_date_sk
+ | GROUP BY c_customer_id,
+ | c_first_name,
+ | c_last_name,
+ | c_preferred_cust_flag,
+ | c_birth_country,
+ | c_login,
+ | c_email_address,
+ | d_year
+ | UNION ALL
+ | SELECT c_customer_id customer_id
+ | ,c_first_name customer_first_name
+ | ,c_last_name customer_last_name
+ | ,c_preferred_cust_flag customer_preferred_cust_flag
+ | ,c_birth_country customer_birth_country
+ | ,c_login customer_login
+ | ,c_email_address customer_email_address
+ | ,d_year dyear
+ | ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)
+ | +ws_ext_sales_price)/2) ) year_total
+ | ,'w' sale_type
+ | FROM customer, web_sales, date_dim
+ | WHERE c_customer_sk = ws_bill_customer_sk AND ws_sold_date_sk = d_date_sk
+ | GROUP BY c_customer_id,
+ | c_first_name,
+ | c_last_name,
+ | c_preferred_cust_flag,
+ | c_birth_country,
+ | c_login,
+ | c_email_address,
+ | d_year)
+ | SELECT
+ | t_s_secyear.customer_id,
+ | t_s_secyear.customer_first_name,
+ | t_s_secyear.customer_last_name,
+ | t_s_secyear.customer_preferred_cust_flag,
+ | t_s_secyear.customer_birth_country,
+ | t_s_secyear.customer_login,
+ | t_s_secyear.customer_email_address
+ | FROM year_total t_s_firstyear, year_total t_s_secyear, year_total t_c_firstyear,
+ | year_total t_c_secyear, year_total t_w_firstyear, year_total t_w_secyear
+ | WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id
+ | and t_s_firstyear.customer_id = t_c_secyear.customer_id
+ | and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+ | and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ | and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ | and t_s_firstyear.sale_type = 's'
+ | and t_c_firstyear.sale_type = 'c'
+ | and t_w_firstyear.sale_type = 'w'
+ | and t_s_secyear.sale_type = 's'
+ | and t_c_secyear.sale_type = 'c'
+ | and t_w_secyear.sale_type = 'w'
+ | and t_s_firstyear.dyear = 2001
+ | and t_s_secyear.dyear = 2001+1
+ | and t_c_firstyear.dyear = 2001
+ | and t_c_secyear.dyear = 2001+1
+ | and t_w_firstyear.dyear = 2001
+ | and t_w_secyear.dyear = 2001+1
+ | and t_s_firstyear.year_total > 0
+ | and t_c_firstyear.year_total > 0
+ | and t_w_firstyear.year_total > 0
+ | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total /
+ | t_c_firstyear.year_total else null end
+ | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total /
+ | t_s_firstyear.year_total else null end
+ | and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total /
+ | t_c_firstyear.year_total else null end
+ | > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total /
+ | t_w_firstyear.year_total else null end
+ | ORDER BY
+ | t_s_secyear.customer_id,
+ | t_s_secyear.customer_first_name,
+ | t_s_secyear.customer_last_name,
+ | t_s_secyear.customer_preferred_cust_flag,
+ | t_s_secyear.customer_birth_country,
+ | t_s_secyear.customer_login,
+ | t_s_secyear.customer_email_address
+ | LIMIT 100
+ """.stripMargin),
+ // Modifications: "+ days" -> date_add
+ // Modifications: "||" -> concat
+ ("q5",
+ """
+ | WITH ssr AS
+ | (SELECT s_store_id,
+ | sum(sales_price) as sales,
+ | sum(profit) as profit,
+ | sum(return_amt) as returns,
+ | sum(net_loss) as profit_loss
+ | FROM
+ | (SELECT ss_store_sk as store_sk,
+ | ss_sold_date_sk as date_sk,
+ | ss_ext_sales_price as sales_price,
+ | ss_net_profit as profit,
+ | cast(0 as decimal(7,2)) as return_amt,
+ | cast(0 as decimal(7,2)) as net_loss
+ | FROM store_sales
+ | UNION ALL
+ | SELECT sr_store_sk as store_sk,
+ | sr_returned_date_sk as date_sk,
+ | cast(0 as decimal(7,2)) as sales_price,
+ | cast(0 as decimal(7,2)) as profit,
+ | sr_return_amt as return_amt,
+ | sr_net_loss as net_loss
+ | FROM store_returns)
+ | salesreturns, date_dim, store
+ | WHERE date_sk = d_date_sk
+ | and d_date between cast('2000-08-23' as date)
+ | and ((cast('2000-08-23' as date) + interval 14 days))
+ | and store_sk = s_store_sk
+ | GROUP BY s_store_id),
+ | csr AS
+ | (SELECT cp_catalog_page_id,
+ | sum(sales_price) as sales,
+ | sum(profit) as profit,
+ | sum(return_amt) as returns,
+ | sum(net_loss) as profit_loss
+ | FROM
+ | (SELECT cs_catalog_page_sk as page_sk,
+ | cs_sold_date_sk as date_sk,
+ | cs_ext_sales_price as sales_price,
+ | cs_net_profit as profit,
+ | cast(0 as decimal(7,2)) as return_amt,
+ | cast(0 as decimal(7,2)) as net_loss
+ | FROM catalog_sales
+ | UNION ALL
+ | SELECT cr_catalog_page_sk as page_sk,
+ | cr_returned_date_sk as date_sk,
+ | cast(0 as decimal(7,2)) as sales_price,
+ | cast(0 as decimal(7,2)) as profit,
+ | cr_return_amount as return_amt,
+ | cr_net_loss as net_loss
+ | from catalog_returns
+ | ) salesreturns, date_dim, catalog_page
+ | WHERE date_sk = d_date_sk
+ | and d_date between cast('2000-08-23' as date)
+ | and ((cast('2000-08-23' as date) + interval 14 days))
+ | and page_sk = cp_catalog_page_sk
+ | GROUP BY cp_catalog_page_id)
+ | ,
+ | wsr AS
+ | (SELECT web_site_id,
+ | sum(sales_price) as sales,
+ | sum(profit) as profit,
+ | sum(return_amt) as returns,
+ | sum(net_loss) as profit_loss
+ | from
+ | (select ws_web_site_sk as wsr_web_site_sk,
+ | ws_sold_date_sk as date_sk,
+ | ws_ext_sales_price as sales_price,
+ | ws_net_profit as profit,
+ | cast(0 as decimal(7,2)) as return_amt,
+ | cast(0 as decimal(7,2)) as net_loss
+ | from web_sales
+ | union all
+ | select ws_web_site_sk as wsr_web_site_sk,
+ | wr_returned_date_sk as date_sk,
+ | cast(0 as decimal(7,2)) as sales_price,
+ | cast(0 as decimal(7,2)) as profit,
+ | wr_return_amt as return_amt,
+ | wr_net_loss as net_loss
+ | FROM web_returns LEFT OUTER JOIN web_sales on
+ | ( wr_item_sk = ws_item_sk
+ | and wr_order_number = ws_order_number)
+ | ) salesreturns, date_dim, web_site
+ | WHERE date_sk = d_date_sk
+ | and d_date between cast('2000-08-23' as date)
+ | and ((cast('2000-08-23' as date) + interval 14 days))
+ | and wsr_web_site_sk = web_site_sk
+ | GROUP BY web_site_id)
+ | SELECT channel,
+ | id,
+ | sum(sales) as sales,
+ | sum(returns) as returns,
+ | sum(profit) as profit
+ | from
+ | (select 'store channel' as channel,
+ | concat('store', s_store_id) as id,
+ | sales,
+ | returns,
+ | (profit - profit_loss) as profit
+ | FROM ssr
+ | UNION ALL
+ | select 'catalog channel' as channel,
+ | concat('catalog_page', cp_catalog_page_id) as id,
+ | sales,
+ | returns,
+ | (profit - profit_loss) as profit
+ | FROM csr
+ | UNION ALL
+ | SELECT 'web channel' as channel,
+ | concat('web_site', web_site_id) as id,
+ | sales,
+ | returns,
+ | (profit - profit_loss) as profit
+ | FROM wsr
+ | ) x
+ | GROUP BY ROLLUP (channel, id)
+ | ORDER BY channel, id
+ | LIMIT 100
+ """.stripMargin),
+ ("q6",
+ """
+ | SELECT a.ca_state state, count(*) cnt
+ | FROM
+ | customer_address a, customer c, store_sales s, date_dim d, item i
+ | WHERE a.ca_address_sk = c.c_current_addr_sk
+ | AND c.c_customer_sk = s.ss_customer_sk
+ | AND s.ss_sold_date_sk = d.d_date_sk
+ | AND s.ss_item_sk = i.i_item_sk
+ | AND d.d_month_seq =
+ | (SELECT distinct (d_month_seq) FROM date_dim
+ | WHERE d_year = 2000 AND d_moy = 1)
+ | AND i.i_current_price > 1.2 *
+ | (SELECT avg(j.i_current_price) FROM item j
+ | WHERE j.i_category = i.i_category)
+ | GROUP BY a.ca_state
+ | HAVING count(*) >= 10
+ | ORDER BY cnt LIMIT 100
+ """.stripMargin),
+ ("q7",
+ """
+ | SELECT i_item_id,
+ | avg(ss_quantity) agg1,
+ | avg(ss_list_price) agg2,
+ | avg(ss_coupon_amt) agg3,
+ | avg(ss_sales_price) agg4
+ | FROM store_sales, customer_demographics, date_dim, item, promotion
+ | WHERE ss_sold_date_sk = d_date_sk AND
+ | ss_item_sk = i_item_sk AND
+ | ss_cdemo_sk = cd_demo_sk AND
+ | ss_promo_sk = p_promo_sk AND
+ | cd_gender = 'M' AND
+ | cd_marital_status = 'S' AND
+ | cd_education_status = 'College' AND
+ | (p_channel_email = 'N' or p_channel_event = 'N') AND
+ | d_year = 2000
+ | GROUP BY i_item_id
+ | ORDER BY i_item_id LIMIT 100
+ """.stripMargin),
+ ("q8",
+ """
+ | select s_store_name, sum(ss_net_profit)
+ | from store_sales, date_dim, store,
+ | (SELECT ca_zip
+ | from (
+ | (SELECT substr(ca_zip,1,5) ca_zip FROM customer_address
+ | WHERE substr(ca_zip,1,5) IN (
+ | '24128','76232','65084','87816','83926','77556','20548',
+ | '26231','43848','15126','91137','61265','98294','25782',
+ | '17920','18426','98235','40081','84093','28577','55565',
+ | '17183','54601','67897','22752','86284','18376','38607',
+ | '45200','21756','29741','96765','23932','89360','29839',
+ | '25989','28898','91068','72550','10390','18845','47770',
+ | '82636','41367','76638','86198','81312','37126','39192',
+ | '88424','72175','81426','53672','10445','42666','66864',
+ | '66708','41248','48583','82276','18842','78890','49448',
+ | '14089','38122','34425','79077','19849','43285','39861',
+ | '66162','77610','13695','99543','83444','83041','12305',
+ | '57665','68341','25003','57834','62878','49130','81096',
+ | '18840','27700','23470','50412','21195','16021','76107',
+ | '71954','68309','18119','98359','64544','10336','86379',
+ | '27068','39736','98569','28915','24206','56529','57647',
+ | '54917','42961','91110','63981','14922','36420','23006',
+ | '67467','32754','30903','20260','31671','51798','72325',
+ | '85816','68621','13955','36446','41766','68806','16725',
+ | '15146','22744','35850','88086','51649','18270','52867',
+ | '39972','96976','63792','11376','94898','13595','10516',
+ | '90225','58943','39371','94945','28587','96576','57855',
+ | '28488','26105','83933','25858','34322','44438','73171',
+ | '30122','34102','22685','71256','78451','54364','13354',
+ | '45375','40558','56458','28286','45266','47305','69399',
+ | '83921','26233','11101','15371','69913','35942','15882',
+ | '25631','24610','44165','99076','33786','70738','26653',
+ | '14328','72305','62496','22152','10144','64147','48425',
+ | '14663','21076','18799','30450','63089','81019','68893',
+ | '24996','51200','51211','45692','92712','70466','79994',
+ | '22437','25280','38935','71791','73134','56571','14060',
+ | '19505','72425','56575','74351','68786','51650','20004',
+ | '18383','76614','11634','18906','15765','41368','73241',
+ | '76698','78567','97189','28545','76231','75691','22246',
+ | '51061','90578','56691','68014','51103','94167','57047',
+ | '14867','73520','15734','63435','25733','35474','24676',
+ | '94627','53535','17879','15559','53268','59166','11928',
+ | '59402','33282','45721','43933','68101','33515','36634',
+ | '71286','19736','58058','55253','67473','41918','19515',
+ | '36495','19430','22351','77191','91393','49156','50298',
+ | '87501','18652','53179','18767','63193','23968','65164',
+ | '68880','21286','72823','58470','67301','13394','31016',
+ | '70372','67030','40604','24317','45748','39127','26065',
+ | '77721','31029','31880','60576','24671','45549','13376',
+ | '50016','33123','19769','22927','97789','46081','72151',
+ | '15723','46136','51949','68100','96888','64528','14171',
+ | '79777','28709','11489','25103','32213','78668','22245',
+ | '15798','27156','37930','62971','21337','51622','67853',
+ | '10567','38415','15455','58263','42029','60279','37125',
+ | '56240','88190','50308','26859','64457','89091','82136',
+ | '62377','36233','63837','58078','17043','30010','60099',
+ | '28810','98025','29178','87343','73273','30469','64034',
+ | '39516','86057','21309','90257','67875','40162','11356',
+ | '73650','61810','72013','30431','22461','19512','13375',
+ | '55307','30625','83849','68908','26689','96451','38193',
+ | '46820','88885','84935','69035','83144','47537','56616',
+ | '94983','48033','69952','25486','61547','27385','61860',
+ | '58048','56910','16807','17871','35258','31387','35458',
+ | '35576'))
+ | INTERSECT
+ | (select ca_zip
+ | FROM
+ | (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt
+ | FROM customer_address, customer
+ | WHERE ca_address_sk = c_current_addr_sk and
+ | c_preferred_cust_flag='Y'
+ | group by ca_zip
+ | having count(*) > 10) A1)
+ | ) A2
+ | ) V1
+ | where ss_store_sk = s_store_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_qoy = 2 and d_year = 1998
+ | and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2))
+ | group by s_store_name
+ | order by s_store_name LIMIT 100
+ """.stripMargin),
+ ("q9",
+ s"""
+ |select case when (select count(*) from store_sales
+ | where ss_quantity between 1 and 20) > ${ rc(0) }
+ | then (select avg(ss_ext_discount_amt) from store_sales
+ | where ss_quantity between 1 and 20)
+ | else (select avg(ss_net_paid) from store_sales
+ | where ss_quantity between 1 and 20) end bucket1 ,
+ | case when (select count(*) from store_sales
+ | where ss_quantity between 21 and 40) > ${ rc(1) }
+ | then (select avg(ss_ext_discount_amt) from store_sales
+ | where ss_quantity between 21 and 40)
+ | else (select avg(ss_net_paid) from store_sales
+ | where ss_quantity between 21 and 40) end bucket2,
+ | case when (select count(*) from store_sales
+ | where ss_quantity between 41 and 60) > ${ rc(2) }
+ | then (select avg(ss_ext_discount_amt) from store_sales
+ | where ss_quantity between 41 and 60)
+ | else (select avg(ss_net_paid) from store_sales
+ | where ss_quantity between 41 and 60) end bucket3,
+ | case when (select count(*) from store_sales
+ | where ss_quantity between 61 and 80) > ${ rc(3) }
+ | then (select avg(ss_ext_discount_amt) from store_sales
+ | where ss_quantity between 61 and 80)
+ | else (select avg(ss_net_paid) from store_sales
+ | where ss_quantity between 61 and 80) end bucket4,
+ | case when (select count(*) from store_sales
+ | where ss_quantity between 81 and 100) > ${ rc(4) }
+ | then (select avg(ss_ext_discount_amt) from store_sales
+ | where ss_quantity between 81 and 100)
+ | else (select avg(ss_net_paid) from store_sales
+ | where ss_quantity between 81 and 100) end bucket5
+ |from reason
+ |where r_reason_sk = 1
+ """.stripMargin),
+ ("q10",
+ """
+ | select
+ | cd_gender, cd_marital_status, cd_education_status, count(*) cnt1,
+ | cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3,
+ | cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5,
+ | cd_dep_college_count, count(*) cnt6
+ | from
+ | customer c, customer_address ca, customer_demographics
+ | where
+ | c.c_current_addr_sk = ca.ca_address_sk and
+ | ca_county in ('Rush County','Toole County','Jefferson County',
+ | 'Dona Ana County','La Porte County') and
+ | cd_demo_sk = c.c_current_cdemo_sk AND
+ | exists (select * from store_sales, date_dim
+ | where c.c_customer_sk = ss_customer_sk AND
+ | ss_sold_date_sk = d_date_sk AND
+ | d_year = 2002 AND
+ | d_moy between 1 AND 1+3) AND
+ | (exists (select * from web_sales, date_dim
+ | where c.c_customer_sk = ws_bill_customer_sk AND
+ | ws_sold_date_sk = d_date_sk AND
+ | d_year = 2002 AND
+ | d_moy between 1 AND 1+3) or
+ | exists (select * from catalog_sales, date_dim
+ | where c.c_customer_sk = cs_ship_customer_sk AND
+ | cs_sold_date_sk = d_date_sk AND
+ | d_year = 2002 AND
+ | d_moy between 1 AND 1+3))
+ | group by cd_gender,
+ | cd_marital_status,
+ | cd_education_status,
+ | cd_purchase_estimate,
+ | cd_credit_rating,
+ | cd_dep_count,
+ | cd_dep_employed_count,
+ | cd_dep_college_count
+ | order by cd_gender,
+ | cd_marital_status,
+ | cd_education_status,
+ | cd_purchase_estimate,
+ | cd_credit_rating,
+ | cd_dep_count,
+ | cd_dep_employed_count,
+ | cd_dep_college_count
+ |LIMIT 100
+ """.stripMargin),
+ ("q11",
+ """
+ | with year_total as (
+ | select c_customer_id customer_id
+ | ,c_first_name customer_first_name
+ | ,c_last_name customer_last_name
+ | ,c_preferred_cust_flag customer_preferred_cust_flag
+ | ,c_birth_country customer_birth_country
+ | ,c_login customer_login
+ | ,c_email_address customer_email_address
+ | ,d_year dyear
+ | ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total
+ | ,'s' sale_type
+ | from customer, store_sales, date_dim
+ | where c_customer_sk = ss_customer_sk
+ | and ss_sold_date_sk = d_date_sk
+ | group by c_customer_id
+ | ,c_first_name
+ | ,c_last_name
+ | ,d_year
+ | ,c_preferred_cust_flag
+ | ,c_birth_country
+ | ,c_login
+ | ,c_email_address
+ | ,d_year
+ | union all
+ | select c_customer_id customer_id
+ | ,c_first_name customer_first_name
+ | ,c_last_name customer_last_name
+ | ,c_preferred_cust_flag customer_preferred_cust_flag
+ | ,c_birth_country customer_birth_country
+ | ,c_login customer_login
+ | ,c_email_address customer_email_address
+ | ,d_year dyear
+ | ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total
+ | ,'w' sale_type
+ | from customer, web_sales, date_dim
+ | where c_customer_sk = ws_bill_customer_sk
+ | and ws_sold_date_sk = d_date_sk
+ | group by
+ | c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country,
+ | c_login, c_email_address, d_year)
+ | select
+ | t_s_secyear.customer_preferred_cust_flag
+ | from year_total t_s_firstyear
+ | ,year_total t_s_secyear
+ | ,year_total t_w_firstyear
+ | ,year_total t_w_secyear
+ | where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ | and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ | and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ | and t_s_firstyear.sale_type = 's'
+ | and t_w_firstyear.sale_type = 'w'
+ | and t_s_secyear.sale_type = 's'
+ | and t_w_secyear.sale_type = 'w'
+ | and t_s_firstyear.dyear = 2001
+ | and t_s_secyear.dyear = 2001+1
+ | and t_w_firstyear.dyear = 2001
+ | and t_w_secyear.dyear = 2001+1
+ | and t_s_firstyear.year_total > 0
+ | and t_w_firstyear.year_total > 0
+ | and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total /
+ | t_w_firstyear.year_total else null end
+ | > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total /
+ | t_s_firstyear.year_total else null end
+ | order by t_s_secyear.customer_preferred_cust_flag
+ | LIMIT 100
+ """.stripMargin),
+ // Modifications: "+ days" -> date_add
+ ("q12",
+ """
+ | select
+ | i_item_desc, i_category, i_class, i_current_price,
+ | sum(ws_ext_sales_price) as itemrevenue,
+ | sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over
+ | (partition by i_class) as revenueratio
+ | from
+ | web_sales, item, date_dim
+ | where
+ | ws_item_sk = i_item_sk
+ | and i_category in ('Sports', 'Books', 'Home')
+ | and ws_sold_date_sk = d_date_sk
+ | and d_date between cast('1999-02-22' as date)
+ | and (cast('1999-02-22' as date) + interval 30 days)
+ | group by
+ | i_item_id, i_item_desc, i_category, i_class, i_current_price
+ | order by
+ | i_category, i_class, i_item_id, i_item_desc, revenueratio
+ | LIMIT 100
+ """.stripMargin),
+ ("q13",
+ """
+ | select avg(ss_quantity)
+ | ,avg(ss_ext_sales_price)
+ | ,avg(ss_ext_wholesale_cost)
+ | ,sum(ss_ext_wholesale_cost)
+ | from store_sales
+ | ,store
+ | ,customer_demographics
+ | ,household_demographics
+ | ,customer_address
+ | ,date_dim
+ | where s_store_sk = ss_store_sk
+ | and ss_sold_date_sk = d_date_sk and d_year = 2001
+ | and((ss_hdemo_sk=hd_demo_sk
+ | and cd_demo_sk = ss_cdemo_sk
+ | and cd_marital_status = 'M'
+ | and cd_education_status = 'Advanced Degree'
+ | and ss_sales_price between 100.00 and 150.00
+ | and hd_dep_count = 3
+ | )or
+ | (ss_hdemo_sk=hd_demo_sk
+ | and cd_demo_sk = ss_cdemo_sk
+ | and cd_marital_status = 'S'
+ | and cd_education_status = 'College'
+ | and ss_sales_price between 50.00 and 100.00
+ | and hd_dep_count = 1
+ | ) or
+ | (ss_hdemo_sk=hd_demo_sk
+ | and cd_demo_sk = ss_cdemo_sk
+ | and cd_marital_status = 'W'
+ | and cd_education_status = '2 yr Degree'
+ | and ss_sales_price between 150.00 and 200.00
+ | and hd_dep_count = 1
+ | ))
+ | and((ss_addr_sk = ca_address_sk
+ | and ca_country = 'United States'
+ | and ca_state in ('TX', 'OH', 'TX')
+ | and ss_net_profit between 100 and 200
+ | ) or
+ | (ss_addr_sk = ca_address_sk
+ | and ca_country = 'United States'
+ | and ca_state in ('OR', 'NM', 'KY')
+ | and ss_net_profit between 150 and 300
+ | ) or
+ | (ss_addr_sk = ca_address_sk
+ | and ca_country = 'United States'
+ | and ca_state in ('VA', 'TX', 'MS')
+ | and ss_net_profit between 50 and 250
+ | ))
+ """.stripMargin),
+ ("q14a",
+ """
+ |with cross_items as
+ | (select i_item_sk ss_item_sk
+ | from item,
+ | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id
+ | category_id
+ | from store_sales, item iss, date_dim d1
+ | where ss_item_sk = iss.i_item_sk
+ and ss_sold_date_sk = d1.d_date_sk
+ | and d1.d_year between 1999 AND 1999 + 2
+ | intersect
+ | select ics.i_brand_id, ics.i_class_id, ics.i_category_id
+ | from catalog_sales, item ics, date_dim d2
+ | where cs_item_sk = ics.i_item_sk
+ | and cs_sold_date_sk = d2.d_date_sk
+ | and d2.d_year between 1999 AND 1999 + 2
+ | intersect
+ | select iws.i_brand_id, iws.i_class_id, iws.i_category_id
+ | from web_sales, item iws, date_dim d3
+ | where ws_item_sk = iws.i_item_sk
+ | and ws_sold_date_sk = d3.d_date_sk
+ | and d3.d_year between 1999 AND 1999 + 2) x
+ | where i_brand_id = brand_id
+ | and i_class_id = class_id
+ | and i_category_id = category_id
+ |),
+ | avg_sales as
+ | (select avg(quantity*list_price) average_sales
+ | from (
+ | select ss_quantity quantity, ss_list_price list_price
+ | from store_sales, date_dim
+ | where ss_sold_date_sk = d_date_sk
+ | and d_year between 1999 and 2001
+ | union all
+ | select cs_quantity quantity, cs_list_price list_price
+ | from catalog_sales, date_dim
+ | where cs_sold_date_sk = d_date_sk
+ | and d_year between 1999 and 1999 + 2
+ | union all
+ | select ws_quantity quantity, ws_list_price list_price
+ | from web_sales, date_dim
+ | where ws_sold_date_sk = d_date_sk
+ | and d_year between 1999 and 1999 + 2) x)
+ | select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales)
+ | from(
+ | select 'store' channel, i_brand_id,i_class_id
+ | ,i_category_id,sum(ss_quantity*ss_list_price) sales
+ | , count(*) number_sales
+ | from store_sales, item, date_dim
+ | where ss_item_sk in (select ss_item_sk from cross_items)
+ | and ss_item_sk = i_item_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_year = 1999+2
+ | and d_moy = 11
+ | group by i_brand_id,i_class_id,i_category_id
+ | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)
+ | union all
+ | select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum
+ | (cs_quantity*cs_list_price) sales, count(*) number_sales
+ | from catalog_sales, item, date_dim
+ | where cs_item_sk in (select ss_item_sk from cross_items)
+ | and cs_item_sk = i_item_sk
+ | and cs_sold_date_sk = d_date_sk
+ | and d_year = 1999+2
+ | and d_moy = 11
+ | group by i_brand_id,i_class_id,i_category_id
+ | having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales)
+ | union all
+ | select 'web' channel, i_brand_id,i_class_id,i_category_id, sum
+ | (ws_quantity*ws_list_price) sales , count(*) number_sales
+ | from web_sales, item, date_dim
+ | where ws_item_sk in (select ss_item_sk from cross_items)
+ | and ws_item_sk = i_item_sk
+ | and ws_sold_date_sk = d_date_sk
+ | and d_year = 1999+2
+ | and d_moy = 11
+ | group by i_brand_id,i_class_id,i_category_id
+ | having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales)
+ | ) y
+ | group by rollup (channel, i_brand_id,i_class_id,i_category_id)
+ | order by channel,i_brand_id,i_class_id,i_category_id
+ | limit 100
+ """.stripMargin),
+ ("q14b",
+ """
+ | with cross_items as
+ | (select i_item_sk ss_item_sk
+ | from item,
+ | (select iss.i_brand_id brand_id, iss.i_class_id class_id, iss.i_category_id
+ | category_id
+ | from store_sales, item iss, date_dim d1
+ | where ss_item_sk = iss.i_item_sk
+ | and ss_sold_date_sk = d1.d_date_sk
+ | and d1.d_year between 1999 AND 1999 + 2
+ | intersect
+ | select ics.i_brand_id, ics.i_class_id, ics.i_category_id
+ | from catalog_sales, item ics, date_dim d2
+ | where cs_item_sk = ics.i_item_sk
+ | and cs_sold_date_sk = d2.d_date_sk
+ | and d2.d_year between 1999 AND 1999 + 2
+ | intersect
+ | select iws.i_brand_id, iws.i_class_id, iws.i_category_id
+ | from web_sales, item iws, date_dim d3
+ | where ws_item_sk = iws.i_item_sk
+ | and ws_sold_date_sk = d3.d_date_sk
+ | and d3.d_year between 1999 AND 1999 + 2) x
+ | where i_brand_id = brand_id
+ | and i_class_id = class_id
+ | and i_category_id = category_id
+ | ),
+ | avg_sales as
+ | (select avg(quantity*list_price) average_sales
+ | from (select ss_quantity quantity, ss_list_price list_price
+ | from store_sales, date_dim
+ | where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2
+ | union all
+ | select cs_quantity quantity, cs_list_price list_price
+ | from catalog_sales, date_dim
+ | where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2
+ | union all
+ | select ws_quantity quantity, ws_list_price list_price
+ | from web_sales, date_dim
+ | where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x)
+ | select * from
+ | (select 'store' channel, i_brand_id,i_class_id,i_category_id
+ | ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales
+ | from store_sales, item, date_dim
+ | where ss_item_sk in (select ss_item_sk from cross_items)
+ | and ss_item_sk = i_item_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_week_seq = (select d_week_seq from date_dim
+ | where d_year = 1999 + 1 and d_moy = 12 and d_dom = 11)
+ | group by i_brand_id,i_class_id,i_category_id
+ | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year,
+ | (select 'store' channel, i_brand_id,i_class_id
+ | ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales
+ | from store_sales, item, date_dim
+ | where ss_item_sk in (select ss_item_sk from cross_items)
+ | and ss_item_sk = i_item_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_week_seq = (select d_week_seq from date_dim
+ | where d_year = 1999 and d_moy = 12 and d_dom = 11)
+ | group by i_brand_id,i_class_id,i_category_id
+ | having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year
+ | where this_year.i_brand_id= last_year.i_brand_id
+ | and this_year.i_class_id = last_year.i_class_id
+ | and this_year.i_category_id = last_year.i_category_id
+ | order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year
+ | .i_category_id
+ | limit 100
+ """.stripMargin),
+ ("q15",
+ """
+ | select ca_zip, sum(cs_sales_price)
+ | from catalog_sales, customer, customer_address, date_dim
+ | where cs_bill_customer_sk = c_customer_sk
+ | and c_current_addr_sk = ca_address_sk
+ | and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475',
+ | '85392', '85460', '80348', '81792')
+ | or ca_state in ('CA','WA','GA')
+ | or cs_sales_price > 500)
+ | and cs_sold_date_sk = d_date_sk
+ | and d_qoy = 2 and d_year = 2001
+ | group by ca_zip
+ | order by ca_zip
+ | limit 100
+ """.stripMargin),
+ // Modifications: " -> `
+ ("q16",
+ """
+ | select
+ | count(distinct cs_order_number) as `order count`,
+ | sum(cs_ext_ship_cost) as `total shipping cost`,
+ | sum(cs_net_profit) as `total net profit`
+ | from
+ | catalog_sales cs1, date_dim, customer_address, call_center
+ | where
+ | d_date between '2002-02-01' and (cast('2002-02-01' as date) + interval 60 days)
+ | and cs1.cs_ship_date_sk = d_date_sk
+ | and cs1.cs_ship_addr_sk = ca_address_sk
+ | and ca_state = 'GA'
+ | and cs1.cs_call_center_sk = cc_call_center_sk
+ | and cc_county in ('Williamson County','Williamson County','Williamson County',
+ | 'Williamson County', 'Williamson County')
+ | and exists (select *
+ | from catalog_sales cs2
+ | where cs1.cs_order_number = cs2.cs_order_number
+ | and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk)
+ | and not exists(select *
+ | from catalog_returns cr1
+ | where cs1.cs_order_number = cr1.cr_order_number)
+ | order by count(distinct cs_order_number)
+ | limit 100
+ """.stripMargin),
+ ("q17",
+ """
+ | select i_item_id
+ | ,i_item_desc
+ | ,s_state
+ | ,count(ss_quantity) as store_sales_quantitycount
+ | ,avg(ss_quantity) as store_sales_quantityave
+ | ,stddev_samp(ss_quantity) as store_sales_quantitystdev
+ | ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov
+ | ,count(sr_return_quantity) as_store_returns_quantitycount
+ | ,avg(sr_return_quantity) as_store_returns_quantityave
+ | ,stddev_samp(sr_return_quantity) as_store_returns_quantitystdev
+ | ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as
+ | store_returns_quantitycov
+ | ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as
+ | catalog_sales_quantityave
+ | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitystdev
+ | ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov
+ | from store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3,
+ | store, item
+ | where d1.d_quarter_name = '2001Q1'
+ | and d1.d_date_sk = ss_sold_date_sk
+ | and i_item_sk = ss_item_sk
+ | and s_store_sk = ss_store_sk
+ | and ss_customer_sk = sr_customer_sk
+ | and ss_item_sk = sr_item_sk
+ | and ss_ticket_number = sr_ticket_number
+ | and sr_returned_date_sk = d2.d_date_sk
+ | and d2.d_quarter_name in ('2001Q1','2001Q2','2001Q3')
+ | and sr_customer_sk = cs_bill_customer_sk
+ | and sr_item_sk = cs_item_sk
+ | and cs_sold_date_sk = d3.d_date_sk
+ | and d3.d_quarter_name in ('2001Q1','2001Q2','2001Q3')
+ | group by i_item_id, i_item_desc, s_state
+ | order by i_item_id, i_item_desc, s_state
+ | limit 100
+ """.stripMargin),
+ // Modifications: "numeric" -> "decimal"
+ ("q18",
+ """
+ | select i_item_id,
+ | ca_country,
+ | ca_state,
+ | ca_county,
+ | avg( cast(cs_quantity as decimal(12,2))) agg1,
+ | avg( cast(cs_list_price as decimal(12,2))) agg2,
+ | avg( cast(cs_coupon_amt as decimal(12,2))) agg3,
+ | avg( cast(cs_sales_price as decimal(12,2))) agg4,
+ | avg( cast(cs_net_profit as decimal(12,2))) agg5,
+ | avg( cast(c_birth_year as decimal(12,2))) agg6,
+ | avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7
+ | from catalog_sales, customer_demographics cd1,
+ | customer_demographics cd2, customer, customer_address, date_dim, item
+ | where cs_sold_date_sk = d_date_sk and
+ | cs_item_sk = i_item_sk and
+ | cs_bill_cdemo_sk = cd1.cd_demo_sk and
+ | cs_bill_customer_sk = c_customer_sk and
+ | cd1.cd_gender = 'F' and
+ | cd1.cd_education_status = 'Unknown' and
+ | c_current_cdemo_sk = cd2.cd_demo_sk and
+ | c_current_addr_sk = ca_address_sk and
+ | c_birth_month in (1,6,8,9,12,2) and
+ | d_year = 1998 and
+ | ca_state in ('MS','IN','ND','OK','NM','VA','MS')
+ | group by rollup (i_item_id, ca_country, ca_state, ca_county)
+ | order by ca_country, ca_state, ca_county, i_item_id
+ | LIMIT 100
+ """.stripMargin),
+ ("q19",
+ """
+ | select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact,
+ | sum(ss_ext_sales_price) ext_price
+ | from date_dim, store_sales, item,customer,customer_address,store
+ | where d_date_sk = ss_sold_date_sk
+ | and ss_item_sk = i_item_sk
+ | and i_manager_id = 8
+ | and d_moy = 11
+ | and d_year = 1998
+ | and ss_customer_sk = c_customer_sk
+ | and c_current_addr_sk = ca_address_sk
+ | and substr(ca_zip,1,5) <> substr(s_zip,1,5)
+ | and ss_store_sk = s_store_sk
+ | group by i_brand, i_brand_id, i_manufact_id, i_manufact
+ | order by ext_price desc, brand, brand_id, i_manufact_id, i_manufact
+ | limit 100
+ """.stripMargin),
+ ("q20",
+ """
+ |select i_item_desc
+ | ,i_category
+ | ,i_class
+ | ,i_current_price
+ | ,sum(cs_ext_sales_price) as itemrevenue
+ | ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over
+ | (partition by i_class) as revenueratio
+ | from catalog_sales, item, date_dim
+ | where cs_item_sk = i_item_sk
+ | and i_category in ('Sports', 'Books', 'Home')
+ | and cs_sold_date_sk = d_date_sk
+ | and d_date between cast('1999-02-22' as date)
+ | and (cast('1999-02-22' as date) + interval 30 days)
+ | group by i_item_id, i_item_desc, i_category, i_class, i_current_price
+ | order by i_category, i_class, i_item_id, i_item_desc, revenueratio
+ | limit 100
+ """.stripMargin),
+ // Modifications: "+ days" -> date_add
+ ("q21",
+ """
+ | select * from(
+ | select w_warehouse_name, i_item_id,
+ | sum(case when (cast(d_date as date) < cast ('2000-03-11' as date))
+ | then inv_quantity_on_hand
+ | else 0 end) as inv_before,
+ | sum(case when (cast(d_date as date) >= cast ('2000-03-11' as date))
+ | then inv_quantity_on_hand
+ | else 0 end) as inv_after
+ | from inventory, warehouse, item, date_dim
+ | where i_current_price between 0.99 and 1.49
+ | and i_item_sk = inv_item_sk
+ | and inv_warehouse_sk = w_warehouse_sk
+ | and inv_date_sk = d_date_sk
+ | and d_date between (cast('2000-03-11' as date) - interval 30 days)
+ | and (cast('2000-03-11' as date) + interval 30 days)
+ | group by w_warehouse_name, i_item_id) x
+ | where (case when inv_before > 0
+ | then inv_after / inv_before
+ | else null
+ | end) between 2.0/3.0 and 3.0/2.0
+ | order by w_warehouse_name, i_item_id
+ | limit 100
+ """.stripMargin),
+ ("q22",
+ """
+ | select i_product_name, i_brand, i_class, i_category, avg(inv_quantity_on_hand) qoh
+ | from inventory, date_dim, item, warehouse
+ | where inv_date_sk=d_date_sk
+ | and inv_item_sk=i_item_sk
+ | and inv_warehouse_sk = w_warehouse_sk
+ | and d_month_seq between 1200 and 1200 + 11
+ | group by rollup(i_product_name, i_brand, i_class, i_category)
+ | order by qoh, i_product_name, i_brand, i_class, i_category
+ | limit 100
+ """.stripMargin),
+ ("q23a",
+ """
+ | with frequent_ss_items as
+ | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
+ | from store_sales, date_dim, item
+ | where ss_sold_date_sk = d_date_sk
+ | and ss_item_sk = i_item_sk
+ | and d_year in (2000, 2000+1, 2000+2,2000+3)
+ | group by substr(i_item_desc,1,30),i_item_sk,d_date
+ | having count(*) >4),
+ | max_store_sales as
+ | (select max(csales) tpcds_cmax
+ | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales
+ | from store_sales, customer, date_dim
+ | where ss_customer_sk = c_customer_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_year in (2000, 2000+1, 2000+2,2000+3)
+ | group by c_customer_sk) x),
+ | best_ss_customer as
+ | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales
+ | from store_sales, customer
+ | where ss_customer_sk = c_customer_sk
+ | group by c_customer_sk
+ | having sum(ss_quantity*ss_sales_price) > (50/100.0) *
+ | (select * from max_store_sales))
+ | select sum(sales)
+ | from ((select cs_quantity*cs_list_price sales
+ | from catalog_sales, date_dim
+ | where d_year = 2000
+ | and d_moy = 2
+ | and cs_sold_date_sk = d_date_sk
+ | and cs_item_sk in (select item_sk from frequent_ss_items)
+ | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer))
+ | union all
+ | (select ws_quantity*ws_list_price sales
+ | from web_sales, date_dim
+ | where d_year = 2000
+ | and d_moy = 2
+ | and ws_sold_date_sk = d_date_sk
+ | and ws_item_sk in (select item_sk from frequent_ss_items)
+ | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer))) y
+ | limit 100
+ """.stripMargin),
+ ("q23b",
+ """
+ |
+ | with frequent_ss_items as
+ | (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
+ | from store_sales, date_dim, item
+ | where ss_sold_date_sk = d_date_sk
+ | and ss_item_sk = i_item_sk
+ | and d_year in (2000, 2000+1, 2000+2,2000+3)
+ | group by substr(i_item_desc,1,30),i_item_sk,d_date
+ | having count(*) > 4),
+ | max_store_sales as
+ | (select max(csales) tpcds_cmax
+ | from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales
+ | from store_sales, customer, date_dim
+ | where ss_customer_sk = c_customer_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_year in (2000, 2000+1, 2000+2,2000+3)
+ | group by c_customer_sk) x),
+ | best_ss_customer as
+ | (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales
+ | from store_sales
+ | ,customer
+ | where ss_customer_sk = c_customer_sk
+ | group by c_customer_sk
+ | having sum(ss_quantity*ss_sales_price) > (50/100.0) *
+ | (select * from max_store_sales))
+ | select c_last_name,c_first_name,sales
+ | from ((select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales
+ | from catalog_sales, customer, date_dim
+ | where d_year = 2000
+ | and d_moy = 2
+ | and cs_sold_date_sk = d_date_sk
+ | and cs_item_sk in (select item_sk from frequent_ss_items)
+ | and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer)
+ | and cs_bill_customer_sk = c_customer_sk
+ | group by c_last_name,c_first_name)
+ | union all
+ | (select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales
+ | from web_sales, customer, date_dim
+ | where d_year = 2000
+ | and d_moy = 2
+ | and ws_sold_date_sk = d_date_sk
+ | and ws_item_sk in (select item_sk from frequent_ss_items)
+ | and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)
+ | and ws_bill_customer_sk = c_customer_sk
+ | group by c_last_name,c_first_name)) y
+ | order by c_last_name,c_first_name,sales
+ | limit 100
+ """.stripMargin),
+ ("q24a",
+ """
+ | with ssales as
+ | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color,
+ | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid
+ | from store_sales, store_returns, store, item, customer, customer_address
+ | where ss_ticket_number = sr_ticket_number
+ | and ss_item_sk = sr_item_sk
+ | and ss_customer_sk = c_customer_sk
+ | and ss_item_sk = i_item_sk
+ | and ss_store_sk = s_store_sk
+ | and c_birth_country = upper(ca_country)
+ | and s_zip = ca_zip
+ | and s_market_id = 8
+ | group by c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color,
+ | i_current_price, i_manager_id, i_units, i_size)
+ | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid
+ | from ssales
+ | where i_color = 'pale'
+ | group by c_last_name, c_first_name, s_store_name
+ | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales)
+ """.stripMargin),
+ ("q24b",
+ """
+ | with ssales as
+ | (select c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color,
+ | i_current_price, i_manager_id, i_units, i_size, sum(ss_net_paid) netpaid
+ | from store_sales, store_returns, store, item, customer, customer_address
+ | where ss_ticket_number = sr_ticket_number
+ | and ss_item_sk = sr_item_sk
+ | and ss_customer_sk = c_customer_sk
+ | and ss_item_sk = i_item_sk
+ | and ss_store_sk = s_store_sk
+ | and c_birth_country = upper(ca_country)
+ | and s_zip = ca_zip
+ | and s_market_id = 8
+ | group by c_last_name, c_first_name, s_store_name, ca_state, s_state,
+ | i_color, i_current_price, i_manager_id, i_units, i_size)
+ | select c_last_name, c_first_name, s_store_name, sum(netpaid) paid
+ | from ssales
+ | where i_color = 'chiffon'
+ | group by c_last_name, c_first_name, s_store_name
+ | having sum(netpaid) > (select 0.05*avg(netpaid) from ssales)
+ """.stripMargin),
+ ("q25",
+ """
+ | select i_item_id, i_item_desc, s_store_id, s_store_name,
+ | sum(ss_net_profit) as store_sales_profit,
+ | sum(sr_net_loss) as store_returns_loss,
+ | sum(cs_net_profit) as catalog_sales_profit
+ | from
+ | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2, date_dim d3,
+ | store, item
+ | where
+ | d1.d_moy = 4
+ | and d1.d_year = 2001
+ | and d1.d_date_sk = ss_sold_date_sk
+ | and i_item_sk = ss_item_sk
+ | and s_store_sk = ss_store_sk
+ | and ss_customer_sk = sr_customer_sk
+ | and ss_item_sk = sr_item_sk
+ | and ss_ticket_number = sr_ticket_number
+ | and sr_returned_date_sk = d2.d_date_sk
+ | and d2.d_moy between 4 and 10
+ | and d2.d_year = 2001
+ | and sr_customer_sk = cs_bill_customer_sk
+ | and sr_item_sk = cs_item_sk
+ | and cs_sold_date_sk = d3.d_date_sk
+ | and d3.d_moy between 4 and 10
+ | and d3.d_year = 2001
+ | group by
+ | i_item_id, i_item_desc, s_store_id, s_store_name
+ | order by
+ | i_item_id, i_item_desc, s_store_id, s_store_name
+ | limit 100
+ """.stripMargin),
+ ("q26",
+ """
+ | select i_item_id,
+ | avg(cs_quantity) agg1,
+ | avg(cs_list_price) agg2,
+ | avg(cs_coupon_amt) agg3,
+ | avg(cs_sales_price) agg4
+ | from catalog_sales, customer_demographics, date_dim, item, promotion
+ | where cs_sold_date_sk = d_date_sk and
+ | cs_item_sk = i_item_sk and
+ | cs_bill_cdemo_sk = cd_demo_sk and
+ | cs_promo_sk = p_promo_sk and
+ | cd_gender = 'M' and
+ | cd_marital_status = 'S' and
+ | cd_education_status = 'College' and
+ | (p_channel_email = 'N' or p_channel_event = 'N') and
+ | d_year = 2000
+ | group by i_item_id
+ | order by i_item_id
+ | limit 100
+ """.stripMargin),
+ ("q27",
+ """
+ | select i_item_id,
+ | s_state, grouping(s_state) g_state,
+ | avg(ss_quantity) agg1,
+ | avg(ss_list_price) agg2,
+ | avg(ss_coupon_amt) agg3,
+ | avg(ss_sales_price) agg4
+ | from store_sales, customer_demographics, date_dim, store, item
+ | where ss_sold_date_sk = d_date_sk and
+ | ss_item_sk = i_item_sk and
+ | ss_store_sk = s_store_sk and
+ | ss_cdemo_sk = cd_demo_sk and
+ | cd_gender = 'M' and
+ | cd_marital_status = 'S' and
+ | cd_education_status = 'College' and
+ | d_year = 2002 and
+ | s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN')
+ | group by rollup (i_item_id, s_state)
+ | order by i_item_id, s_state
+ | limit 100
+ """.stripMargin),
+ ("q28",
+ """
+ | select *
+ | from (select avg(ss_list_price) B1_LP
+ | ,count(ss_list_price) B1_CNT
+ | ,count(distinct ss_list_price) B1_CNTD
+ | from store_sales
+ | where ss_quantity between 0 and 5
+ | and (ss_list_price between 8 and 8+10
+ | or ss_coupon_amt between 459 and 459+1000
+ | or ss_wholesale_cost between 57 and 57+20)) B1,
+ | (select avg(ss_list_price) B2_LP
+ | ,count(ss_list_price) B2_CNT
+ | ,count(distinct ss_list_price) B2_CNTD
+ | from store_sales
+ | where ss_quantity between 6 and 10
+ | and (ss_list_price between 90 and 90+10
+ | or ss_coupon_amt between 2323 and 2323+1000
+ | or ss_wholesale_cost between 31 and 31+20)) B2,
+ | (select avg(ss_list_price) B3_LP
+ | ,count(ss_list_price) B3_CNT
+ | ,count(distinct ss_list_price) B3_CNTD
+ | from store_sales
+ | where ss_quantity between 11 and 15
+ | and (ss_list_price between 142 and 142+10
+ | or ss_coupon_amt between 12214 and 12214+1000
+ | or ss_wholesale_cost between 79 and 79+20)) B3,
+ | (select avg(ss_list_price) B4_LP
+ | ,count(ss_list_price) B4_CNT
+ | ,count(distinct ss_list_price) B4_CNTD
+ | from store_sales
+ | where ss_quantity between 16 and 20
+ | and (ss_list_price between 135 and 135+10
+ | or ss_coupon_amt between 6071 and 6071+1000
+ | or ss_wholesale_cost between 38 and 38+20)) B4,
+ | (select avg(ss_list_price) B5_LP
+ | ,count(ss_list_price) B5_CNT
+ | ,count(distinct ss_list_price) B5_CNTD
+ | from store_sales
+ | where ss_quantity between 21 and 25
+ | and (ss_list_price between 122 and 122+10
+ | or ss_coupon_amt between 836 and 836+1000
+ | or ss_wholesale_cost between 17 and 17+20)) B5,
+ | (select avg(ss_list_price) B6_LP
+ | ,count(ss_list_price) B6_CNT
+ | ,count(distinct ss_list_price) B6_CNTD
+ | from store_sales
+ | where ss_quantity between 26 and 30
+ | and (ss_list_price between 154 and 154+10
+ | or ss_coupon_amt between 7326 and 7326+1000
+ | or ss_wholesale_cost between 7 and 7+20)) B6
+ | limit 100
+ """.stripMargin),
+ ("q29",
+ """
+ | select
+ | i_item_id
+ | ,i_item_desc
+ | ,s_store_id
+ | ,s_store_name
+ | ,sum(ss_quantity) as store_sales_quantity
+ | ,sum(sr_return_quantity) as store_returns_quantity
+ | ,sum(cs_quantity) as catalog_sales_quantity
+ | from
+ | store_sales, store_returns, catalog_sales, date_dim d1, date_dim d2,
+ | date_dim d3, store, item
+ | where
+ | d1.d_moy = 9
+ | and d1.d_year = 1999
+ | and d1.d_date_sk = ss_sold_date_sk
+ | and i_item_sk = ss_item_sk
+ | and s_store_sk = ss_store_sk
+ | and ss_customer_sk = sr_customer_sk
+ | and ss_item_sk = sr_item_sk
+ | and ss_ticket_number = sr_ticket_number
+ | and sr_returned_date_sk = d2.d_date_sk
+ | and d2.d_moy between 9 and 9 + 3
+ | and d2.d_year = 1999
+ | and sr_customer_sk = cs_bill_customer_sk
+ | and sr_item_sk = cs_item_sk
+ | and cs_sold_date_sk = d3.d_date_sk
+ | and d3.d_year in (1999,1999+1,1999+2)
+ | group by
+ | i_item_id, i_item_desc, s_store_id, s_store_name
+ | order by
+ | i_item_id, i_item_desc, s_store_id, s_store_name
+ | limit 100
+ """.stripMargin),
+ ("q30",
+ """
+ | with customer_total_return as
+ | (select wr_returning_customer_sk as ctr_customer_sk
+ | ,ca_state as ctr_state,
+ | sum(wr_return_amt) as ctr_total_return
+ | from web_returns, date_dim, customer_address
+ | where wr_returned_date_sk = d_date_sk
+ | and d_year = 2002
+ | and wr_returning_addr_sk = ca_address_sk
+ | group by wr_returning_customer_sk,ca_state)
+ | select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag
+ | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address
+ | ,c_last_review_date,ctr_total_return
+ | from customer_total_return ctr1, customer_address, customer
+ | where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2
+ | from customer_total_return ctr2
+ | where ctr1.ctr_state = ctr2.ctr_state)
+ | and ca_address_sk = c_current_addr_sk
+ | and ca_state = 'GA'
+ | and ctr1.ctr_customer_sk = c_customer_sk
+ | order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag
+ | ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,
+ | c_email_address
+ | ,c_last_review_date,ctr_total_return
+ | limit 100
+ """.stripMargin),
+ ("q31",
+ """
+ | with ss as
+ | (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales
+ | from store_sales,date_dim,customer_address
+ | where ss_sold_date_sk = d_date_sk
+ | and ss_addr_sk=ca_address_sk
+ | group by ca_county,d_qoy, d_year),
+ | ws as
+ | (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales
+ | from web_sales,date_dim,customer_address
+ | where ws_sold_date_sk = d_date_sk
+ | and ws_bill_addr_sk=ca_address_sk
+ | group by ca_county,d_qoy, d_year)
+ | select
+ | ss1.ca_county
+ | ,ss1.d_year
+ | ,ws2.web_sales/ws1.web_sales web_q1_q2_increase
+ | ,ss2.store_sales/ss1.store_sales store_q1_q2_increase
+ | ,ws3.web_sales/ws2.web_sales web_q2_q3_increase
+ | ,ss3.store_sales/ss2.store_sales store_q2_q3_increase
+ | from
+ | ss ss1, ss ss2, ss ss3, ws ws1, ws ws2, ws ws3
+ | where
+ | ss1.d_qoy = 1
+ | and ss1.d_year = 2000
+ | and ss1.ca_county = ss2.ca_county
+ | and ss2.d_qoy = 2
+ | and ss2.d_year = 2000
+ | and ss2.ca_county = ss3.ca_county
+ | and ss3.d_qoy = 3
+ | and ss3.d_year = 2000
+ | and ss1.ca_county = ws1.ca_county
+ | and ws1.d_qoy = 1
+ | and ws1.d_year = 2000
+ | and ws1.ca_county = ws2.ca_county
+ | and ws2.d_qoy = 2
+ | and ws2.d_year = 2000
+ | and ws1.ca_county = ws3.ca_county
+ | and ws3.d_qoy = 3
+ | and ws3.d_year = 2000
+ | and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end
+ | > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end
+ | and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end
+ | > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end
+ | order by ss1.ca_county
+ """.stripMargin),
+ // Modifications: " -> `
+ ("q32",
+ """
+ | select sum(cs_ext_discount_amt) as `excess discount amount`
+ | from
+ | catalog_sales, item, date_dim
+ | where
+ | i_manufact_id = 977
+ | and i_item_sk = cs_item_sk
+ | and d_date between '2000-01-27' and (cast('2000-01-27' as date) + interval 90 days)
+ | and d_date_sk = cs_sold_date_sk
+ | and cs_ext_discount_amt > (
+ | select 1.3 * avg(cs_ext_discount_amt)
+ | from catalog_sales, date_dim
+ | where cs_item_sk = i_item_sk
+ | and d_date between '2000-01-27]' and (cast('2000-01-27' as date) + interval
+ | 90 days)
+ | and d_date_sk = cs_sold_date_sk)
+ |limit 100
+ """.stripMargin),
+ ("q33",
+ """
+ | with ss as (
+ | select
+ | i_manufact_id,sum(ss_ext_sales_price) total_sales
+ | from
+ | store_sales, date_dim, customer_address, item
+ | where
+ | i_manufact_id in (select i_manufact_id
+ | from item
+ | where i_category in ('Electronics'))
+ | and ss_item_sk = i_item_sk
+ | and ss_sold_date_sk = d_date_sk
+ | and d_year = 1998
+ | and d_moy = 5
+ | and ss_addr_sk = ca_address_sk
+ | and ca_gmt_offset = -5
+ | group by i_manufact_id), cs as
+ | (select i_manufact_id, sum(cs_ext_sales_price) total_sales
+ | from catalog_sales, date_dim, customer_address, item
+ | where
+ | i_manufact_id in (
+ | select i_manufact_id from item
+ | where
+ | i_category in ('Electronics'))
+ | and cs_item_sk = i_item_sk
+ | and cs_sold_date_sk = d_date_sk
+ | and d_year = 1998
+ | and d_moy = 5
+ | and cs_bill_addr_sk = ca_address_sk
+ | and ca_gmt_offset = -5
+ | group by i_manufact_id),
+ | ws as (
+ | select i_manufact_id,sum(ws_ext_sales_price) total_sales
+ | from
+ | web_sales, date_dim, customer_address, item
+ | where
+ | i_manufact_id in (select i_manufact_id from item
+ | where i_category in ('Electronics'))
+ | and ws_item_sk = i_item_sk
+ | and ws_sold_date_sk = d_date_sk
+ | and d_year = 1998
+ | and d_moy = 5
+ | and ws_bill_addr_sk = ca_address_sk
+ | and ca_gmt_offset = -5
+ | group by i_manufact_id)
+ | select i_manufact_id ,sum(total_sales) total_sales
+ | from (select * from ss
+ | union all
+ | select * from cs
+ | union all
+ | select * from ws) tmp1
+ | group by i_manufact_id
+ | order by total_sales
+ |limit 100
+ """.stripMargin),
+ ("q34",
+ """
+ | select c_last_name, c_first_name, c_salutation, c_preferred_cust_flag, ss_ticket_number,
+ | cnt
+ | FROM
+ | (select ss_ticket_number, ss_customer_sk, count(*) cnt
+ | from store_sales,date_dim,store,household_demographics
+ | where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | and store_sales.ss_store_sk = store.s_store_sk
+ | and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk
+ | and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28)
+ | and (household_demographics.hd_buy_potential = '>10000' or
+ | household_demographics.hd_buy_potential = 'unknown')
+ | and household_demographics.hd_vehicle_count > 0
+ | and (case when household_demographics.hd_vehicle_count > 0
+ | then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count
+ | else null
+ | end) > 1.2
+ | and date_dim.d_year in (1999, 1999+1, 1999+2)
+ | and store.s_county in ('Williamson County','Williamson County','Williamson County',
+ | 'Williamson County',
+ | 'Williamson County','Williamson County','Williamson County',
+ | 'Williamson County')
+ | group by ss_ticket_number,ss_customer_sk) dn,customer
+ | where ss_customer_sk = c_customer_sk
+ | and cnt between 15 and 20
+ | order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc
+ """.stripMargin),
+ ("q35",
+ """
+ | select
+ | ca_state,
+ | cd_gender,
+ | cd_marital_status,
+ | count(*) cnt1,
+ | min(cd_dep_count),
+ | max(cd_dep_count),
+ | avg(cd_dep_count),
+ | cd_dep_employed_count,
+ | count(*) cnt2,
+ | min(cd_dep_employed_count),
+ | max(cd_dep_employed_count),
+ | avg(cd_dep_employed_count),
+ | cd_dep_college_count,
+ | count(*) cnt3,
+ | min(cd_dep_college_count),
+ | max(cd_dep_college_count),
+ | avg(cd_dep_college_count)
+ | from
+ | customer c,customer_address ca,customer_demographics
+ | where
+ | c.c_current_addr_sk = ca.ca_address_sk and
+ | cd_demo_sk = c.c_current_cdemo_sk and
+ | exists (select * from store_sales, date_dim
+ | where c.c_customer_sk = ss_customer_sk and
+ | ss_sold_date_sk = d_date_sk and
+ | d_year = 2002 and
+ | d_qoy < 4) and
+ | (exists (select * from web_sales, date_dim
+ | where c.c_customer_sk = ws_bill_customer_sk and
+ | ws_sold_date_sk = d_date_sk and
+ | d_year = 2002 and
+ | d_qoy < 4) or
+ | exists (select * from catalog_sales, date_dim
+ | where c.c_customer_sk = cs_ship_customer_sk and
+ | cs_sold_date_sk = d_date_sk and
+ | d_year = 2002 and
+ | d_qoy < 4))
+ | group by ca_state, cd_gender, cd_marital_status, cd_dep_count,
+ | cd_dep_employed_count, cd_dep_college_count
+ | order by ca_state, cd_gender, cd_marital_status, cd_dep_count,
+ | cd_dep_employed_count, cd_dep_college_count
+ | limit 100
+ """.stripMargin),
+ ("q36",
+ """
+ | select
+ | sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin
+ | ,i_category
+ | ,i_class
+ | ,grouping(i_category)+grouping(i_class) as lochierarchy
+ | ,rank() over (
+ | partition by grouping(i_category)+grouping(i_class),
+ | case when grouping(i_class) = 0 then i_category end
+ | order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent
+ | from
+ | store_sales, date_dim d1, item, store
+ | where
+ | d1.d_year = 2001
+ | and d1.d_date_sk = ss_sold_date_sk
+ | and i_item_sk = ss_item_sk
+ | and s_store_sk = ss_store_sk
+ | and s_state in ('TN','TN','TN','TN','TN','TN','TN','TN')
+ | group by rollup(i_category,i_class)
+ | order by
+ | lochierarchy desc
+ | ,case when lochierarchy = 0 then i_category end
+ | ,rank_within_parent
+ | limit 100
+ """.stripMargin),
+ // Modifications: "+ days" -> date_add
+ ("q37",
+ """
+ | select i_item_id, i_item_desc, i_current_price
+ | from item, inventory, date_dim, catalog_sales
+ | where i_current_price between 68 and 68 + 30
+ | and inv_item_sk = i_item_sk
+ | and d_date_sk=inv_date_sk
+ | and d_date between cast('2000-02-01' as date) and (cast('2000-02-01' as date) +
+ | interval 60 days)
+ | and i_manufact_id in (677,940,694,808)
+ | and inv_quantity_on_hand between 100 and 500
+ | and cs_item_sk = i_item_sk
+ | group by i_item_id,i_item_desc,i_current_price
+ | order by i_item_id
+ | limit 100
+ """.stripMargin),
+ ("q38",
+ """
+ | select count(*) from (
+ | select distinct c_last_name, c_first_name, d_date
+ | from store_sales, date_dim, customer
+ | where store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ | and store_sales.ss_customer_sk = customer.c_customer_sk
+ | and d_month_seq between 1200 and 1200 + 11
+ | intersect
+ | select distinct c_last_name, c_first_name, d_date
+ | from catalog_sales, date_dim, customer
+ | where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk
+ | and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk
+ | and d_month_seq between 1200 and 1200 + 11
+ | intersect
+ | select distinct c_last_name, c_first_name, d_date
+ | from web_sales, date_dim, customer
+ | where web_sales.ws_sold_date_sk = date_dim.d_date_sk
+ | and web_sales.ws_bill_customer_sk = customer.c_customer_sk
+ | and d_month_seq between 1200 and 1200 + 11
+ | ) hot_cust
+ | limit 100
+ """.stripMargin),
+ ("q39a",
+ """
+ | with inv as
+ | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+ | ,stdev,mean, case mean when 0 then null else stdev/mean end cov
+ | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+ | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean
+ | from inventory, item, warehouse, date_dim
+ | where inv_item_sk = i_item_sk
+ | and inv_warehouse_sk = w_warehouse_sk
+ | and inv_date_sk = d_date_sk
+ | and d_year = 2001
+ | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo
+ | where case mean when 0 then 0 else stdev/mean end > 1)
+ | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov
+ | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov
+ | from inv inv1,inv inv2
+ | where inv1.i_item_sk = inv2.i_item_sk
+ | and inv1.w_warehouse_sk = inv2.w_warehouse_sk
+ | and inv1.d_moy=1
+ | and inv2.d_moy=1+1
+ | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov
+ | ,inv2.d_moy,inv2.mean, inv2.cov
+ """.stripMargin),
+ ("q39b",
+ """
+ | with inv as
+ | (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+ | ,stdev,mean, case mean when 0 then null else stdev/mean end cov
+ | from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy
+ | ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean
+ | from inventory, item, warehouse, date_dim
+ | where inv_item_sk = i_item_sk
+ | and inv_warehouse_sk = w_warehouse_sk
+ | and inv_date_sk = d_date_sk
+ | and d_year = 2001
+ | group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo
+ | where case mean when 0 then 0 else stdev/mean end > 1)
+ | select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov
+ | ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov
+ | from inv inv1,inv inv2
+ | where inv1.i_item_sk = inv2.i_item_sk
+ | and inv1.w_warehouse_sk = inv2.w_warehouse_sk
+ | and inv1.d_moy=1
+ | and inv2.d_moy=1+1
+ | and inv1.cov > 1.5
+ | order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov
+ | ,inv2.d_moy,inv2.mean, inv2.cov
+ """.stripMargin),
+ // Modifications: "+ days" -> date_add
+ ("q40",
+ """
+ | select
+ | w_state
+ | ,i_item_id
+ | ,sum(case when (cast(d_date as date) < cast('2000-03-11' as date))
+ | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before
+ | ,sum(case when (cast(d_date as date) >= cast('2000-03-11' as date))
+ | then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after
+ | from
+ | catalog_sales left outer join catalog_returns on
+ | (cs_order_number = cr_order_number
+ | and cs_item_sk = cr_item_sk)
+ | ,warehouse, item, date_dim
+ | where
+ | i_current_price between 0.99 and 1.49
+ | and i_item_sk = cs_item_sk
+ | and cs_warehouse_sk = w_warehouse_sk
+ | and cs_sold_date_sk = d_date_sk
+ | and d_date between (cast('2000-03-11' as date) - interval 30 days)
+ | and (cast('2000-03-11' as date) + interval 30 days)
+ | group by w_state,i_item_id
+ | order by w_state,i_item_id
+ | limit 100
+ """.stripMargin),
+ ("q41",
+ """
+ | select distinct(i_product_name)
+ | from item i1
+ | where i_manufact_id between 738 and 738+40
+ | and (select count(*) as item_cnt
+ | from item
+ | where (i_manufact = i1.i_manufact and
+ | ((i_category = 'Women' and
+ | (i_color = 'powder' or i_color = 'khaki') and
+ | (i_units = 'Ounce' or i_units = 'Oz') and
+ | (i_size = 'medium' or i_size = 'extra large')
+ | ) or
+ | (i_category = 'Women' and
+ | (i_color = 'brown' or i_color = 'honeydew') and
+ | (i_units = 'Bunch' or i_units = 'Ton') and
+ | (i_size = 'N/A' or i_size = 'small')
+ | ) or
+ | (i_category = 'Men' and
+ | (i_color = 'floral' or i_color = 'deep') and
+ | (i_units = 'N/A' or i_units = 'Dozen') and
+ | (i_size = 'petite' or i_size = 'large')
+ | ) or
+ | (i_category = 'Men' and
+ | (i_color = 'light' or i_color = 'cornflower') and
+ | (i_units = 'Box' or i_units = 'Pound') and
+ | (i_size = 'medium' or i_size = 'extra large')
+ | ))) or
+ | (i_manufact = i1.i_manufact and
+ | ((i_category = 'Women' and
+ | (i_color = 'midnight' or i_color = 'snow') and
+ | (i_units = 'Pallet' or i_units = 'Gross') and
+ | (i_size = 'medium' or i_size = 'extra large')
+ | ) or
+ | (i_category = 'Women' and
+ | (i_color = 'cyan' or i_color = 'papaya') and
+ | (i_units = 'Cup' or i_units = 'Dram') and
+ | (i_size = 'N/A' or i_size = 'small')
+ | ) or
+ | (i_category = 'Men' and
+ | (i_color = 'orange' or i_color = 'frosted') and
+ | (i_units = 'Each' or i_units = 'Tbl') and
+ | (i_size = 'petite' or i_size = 'large')
+ | ) or
+ | (i_category = 'Men' and
+ | (i_color = 'forest' or i_color = 'ghost') and
+ | (i_units = 'Lb' or i_units = 'Bundle') and
+ | (i_size = 'medium' or i_size = 'extra large')
+ | )))) > 0
+ | order by i_product_name
+ | limit 100
+ """.stripMargin),
+ ("q42",
+ """
+ | select dt.d_year, item.i_category_id, item.i_category, sum(ss_ext_sales_price)
+ | from date_dim dt, store_sales, item
+ | where dt.d_date_sk = store_sales.ss_sold_date_sk
+ | and store_sales.ss_item_sk = item.i_item_sk
+ | and item.i_manager_id = 1
+ | and dt.d_moy=11
+ | and dt.d_year=2000
+ | group by dt.d_year
+ | ,item.i_category_id
+ | ,item.i_category
+ | order by sum(ss_ext_sales_price) desc,dt.d_year
+ | ,item.i_category_id
+ | ,item.i_category
+ | limit 100
+ """.stripMargin),
+ ("q43",
+ """
+ | select s_store_name, s_store_id,
+ | sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales,
+ | sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales,
+ | sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales,
+ | sum(case when (d_day_name='Wednesday') then ss_sales_price else null end)
+ | wed_sales,
+ | sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales,
+ | sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales,
+ | sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales
+ | from date_dim, store_sales, store
+ | where d_date_sk = ss_sold_date_sk and
+ | s_store_sk = ss_store_sk and
+ | s_gmt_offset = -5 and
+ | d_year = 2000
+ | group by s_store_name, s_store_id
+ | order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,
+ | thu_sales,fri_sales,sat_sales
+ | limit 100
+ """.stripMargin),
+ ("q44",
+ """
+ | select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing
+ | from(select *
+ | from (select item_sk,rank() over (order by rank_col asc) rnk
+ | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col
+ | from store_sales ss1
+ | where ss_store_sk = 4
+ | group by ss_item_sk
+ | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col
+ | from store_sales
+ | where ss_store_sk = 4
+ | and ss_addr_sk is null
+ | group by ss_store_sk))V1)V11
+ | where rnk < 11) asceding,
+ | (select *
+ | from (select item_sk,rank() over (order by rank_col desc) rnk
+ | from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col
+ | from store_sales ss1
+ | where ss_store_sk = 4
+ | group by ss_item_sk
+ | having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col
+ | from store_sales
+ | where ss_store_sk = 4
+ | and ss_addr_sk is null
+ | group by ss_store_sk))V2)V21
+ | where rnk < 11) descending,
+ | item i1, item i2
+ | where asceding.rnk = descending.rnk
+ | and i1.i_item_sk=asceding.item_sk
+ | and i2.i_item_sk=descending.item_sk
+ |
<TRUNCATED>