You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/04/18 05:09:57 UTC
[impala] branch master updated: IMPALA-11247: Test script changes for materialized views
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 8645ac6db IMPALA-11247: Test script changes for materialized views
8645ac6db is described below
commit 8645ac6db39b30ba6b012b379c2295094e1b427b
Author: Aman Sinha <am...@cloudera.com>
AuthorDate: Sat Apr 16 15:22:14 2022 -0700
IMPALA-11247: Test script changes for materialized views
IMPALA-10723 added support for treating materialized views as tables.
In certain test configurations, the rebuild of the materialized views
(which is done via Hive) was not populating the data in the MV. In
this patch, I have changed the source tables of materialized views
to be full-acid instead of insert-only transactional tables. This
enables the tests to succeed. Insert-only source tables are also
meant to work for the MV rebuild but that is a Hive issue that will
be investigated separately.
Change-Id: I349faa0ad36ec8ca6f574f7f92d9a32fb7d0d344
Reviewed-on: http://gerrit.cloudera.org:8080/18421
Reviewed-by: Aman Sinha <am...@cloudera.com>
Tested-by: Aman Sinha <am...@cloudera.com>
---
.../authorization/AuthorizationStmtTest.java | 6 +--
testdata/bin/compute-table-stats.sh | 3 +-
.../functional/functional_schema_template.sql | 55 ++++------------------
.../datasets/functional/schema_constraints.csv | 6 ---
4 files changed, 12 insertions(+), 58 deletions(-)
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index bb331521f..0bb4e39fd 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -3143,8 +3143,8 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
// Add an unmasked policy. It should not block updates.
createColumnMaskingPolicy("alltypessmall_id_unmask", "functional", "alltypessmall",
"id", user_.getShortName(), "MASK_NONE", /*maskExpr*/null);
- createColumnMaskingPolicy("alltypes_transactional_sint_mask", "functional_orc_def",
- "alltypes_transactional", "smallint_col", user_.getShortName(), "MASK_NULL",
+ createColumnMaskingPolicy("alltypes_sint_mask", "functional_orc_def",
+ "alltypes", "smallint_col", user_.getShortName(), "MASK_NULL",
/*maskExpr*/null);
// Create a column mask on a materialized view column. Since this
// MV references source table with a column masking policy (defined
@@ -3284,7 +3284,7 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
deleteRangerPolicy("alltypestiny_id_mask");
deleteRangerPolicy("kudu_id_mask");
deleteRangerPolicy("alltypessmall_id_unmask");
- deleteRangerPolicy("alltypes_transactional_sint_mask");
+ deleteRangerPolicy("alltypes_sint_mask");
deleteRangerPolicy("mv1_alltypes_jointbl_c2_mask");
}
}
diff --git a/testdata/bin/compute-table-stats.sh b/testdata/bin/compute-table-stats.sh
index b2222581c..c452b4043 100755
--- a/testdata/bin/compute-table-stats.sh
+++ b/testdata/bin/compute-table-stats.sh
@@ -35,8 +35,7 @@ ${COMPUTE_STATS_SCRIPT} --db_names=functional\
--table_names="alltypes,alltypesagg,alltypesaggmultifilesnopart,alltypesaggnonulls,
alltypessmall,alltypestiny,jointbl,dimtbl,stringpartitionkey,nulltable,nullrows,
date_tbl,chars_medium,part_strings_with_quotes,alltypes_date_partition,
- alltypes_date_partition_2,alltypes_transactional,jointbl_transactional,
- mv1_alltypes_jointbl"
+ alltypes_date_partition_2,mv1_alltypes_jointbl"
# We cannot load HBase on s3 and isilon yet.
if [ "${TARGET_FILESYSTEM}" = "hdfs" ]; then
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 3beadf86e..045c57bcb 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2541,32 +2541,6 @@ SELECT * from functional.{table_name};
---- DATASET
functional
---- BASE_TABLE_NAME
-alltypes_transactional
----- CREATE
-CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
- id INT,
- bool_col BOOLEAN,
- tinyint_col TINYINT,
- smallint_col SMALLINT,
- int_col INT,
- bigint_col BIGINT,
- float_col FLOAT,
- double_col DOUBLE,
- date_string_col STRING,
- string_col STRING,
- timestamp_col TIMESTAMP,
- year int,
- month int
-)
-STORED AS {file_format}
-TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only');
----- DEPENDENT_LOAD
-INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name}
-SELECT * FROM {db_name}{db_suffix}.alltypes;
-====
----- DATASET
-functional
----- BASE_TABLE_NAME
materialized_view
---- HIVE_MAJOR_VERSION
3
@@ -3433,23 +3407,6 @@ hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/i
---- DATASET
functional
---- BASE_TABLE_NAME
-jointbl_transactional
----- CREATE
-CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
- test_id BIGINT,
- test_name STRING,
- test_zip INT,
- alltypes_id INT
-)
-STORED AS {file_format}
-TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only');
----- DEPENDENT_LOAD
-INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name}
-SELECT * FROM {db_name}{db_suffix}.jointbl;
-====
----- DATASET
-functional
----- BASE_TABLE_NAME
mv1_alltypes_jointbl
---- HIVE_MAJOR_VERSION
3
@@ -3457,11 +3414,13 @@ mv1_alltypes_jointbl
CREATE MATERIALIZED VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}
STORED AS {file_format} AS SELECT t1.smallint_col c1, t1.bool_col c2,
t2.test_id c3, min(t1.bigint_col) min_bigint, min(t2.test_zip) min_zip
-FROM {db_name}{db_suffix}.alltypes_transactional t1
-JOIN {db_name}{db_suffix}.jointbl_transactional t2 ON (t1.id=t2.alltypes_id)
+FROM {db_name}{db_suffix}.alltypes t1
+JOIN {db_name}{db_suffix}.jointbl t2 ON (t1.id=t2.alltypes_id)
group by t1.smallint_col, t1.bool_col, t2.test_id;
---- DEPENDENT_LOAD_HIVE
ALTER MATERIALIZED VIEW {db_name}{db_suffix}.{table_name} REBUILD;
+-- do a count to confirm if the rebuild populated rows in the MV
+select count(*) as mv_count from {db_name}{db_suffix}.{table_name};
=====
---- DATASET
functional
@@ -3476,9 +3435,11 @@ mv2_alltypes_jointbl
CREATE MATERIALIZED VIEW IF NOT EXISTS {db_name}{db_suffix}.{table_name}
STORED AS {file_format} AS SELECT t1.smallint_col c1, t1.bool_col c2,
t2.test_id c3, max(t1.bigint_col) max_bigint, max(t2.test_zip) max_zip
-FROM {db_name}{db_suffix}.alltypes_transactional t1
-JOIN {db_name}{db_suffix}.jointbl_transactional t2 ON (t1.id=t2.alltypes_id)
+FROM {db_name}{db_suffix}.alltypes t1
+JOIN {db_name}{db_suffix}.jointbl t2 ON (t1.id=t2.alltypes_id)
group by t1.smallint_col, t1.bool_col, t2.test_id;
---- DEPENDENT_LOAD_HIVE
ALTER MATERIALIZED VIEW {db_name}{db_suffix}.{table_name} REBUILD;
+-- do a count to confirm if the rebuild populated rows in the MV
+select count(*) as mv_count from {db_name}{db_suffix}.{table_name};
====
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index 11bcd65d6..c814e4db5 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -297,10 +297,6 @@ table_name:insertonly_nopart_insert, constraint:restrict_to, table_format:text/n
table_name:insertonly_nopart_insert, constraint:restrict_to, table_format:parquet/none/none
table_name:insertonly_part_insert, constraint:restrict_to, table_format:text/none/none
table_name:insertonly_part_insert, constraint:restrict_to, table_format:parquet/none/none
-table_name:alltypes_transactional, constraint:restrict_to, table_format:orc/def/block
-table_name:jointbl_transactional, constraint:restrict_to, table_format:orc/def/block
-table_name:alltypes_transactional, constraint:restrict_to, table_format:parquet/none/none
-table_name:jointbl_transactional, constraint:restrict_to, table_format:parquet/none/none
# A materialized view is based on one or more transactional (in this case insert-only)
# base tables, so the MVs need to be excluded for the table formats where the base
@@ -309,8 +305,6 @@ table_name:materialized_view, constraint:exclude, table_format:hbase/none/none
table_name:materialized_view, constraint:exclude, table_format:kudu/none/none
table_name:mv1_alltypes_jointbl, constraint:restrict_to, table_format:orc/def/block
table_name:mv2_alltypes_jointbl, constraint:restrict_to, table_format:orc/def/block
-table_name:mv1_alltypes_jointbl, constraint:restrict_to, table_format:parquet/none/none
-table_name:mv2_alltypes_jointbl, constraint:restrict_to, table_format:parquet/none/none
table_name:insert_only_transactional_bucketed_table, constraint:exclude, table_format:hbase/none/none
table_name:insert_only_transactional_bucketed_table, constraint:exclude, table_format:kudu/none/none