You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/02 23:33:50 UTC
[impala] branch master updated (ab416d4 -> 79c5f87)
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from ab416d4 IMPALA-8428: Bump CDH_BUILD_NUMBER to 1055188.
new d29d68c IMPALA-8269: Refactor authorization test package structure
new 79c5f87 IMPALA-8121: part 1: some test fixes for catalog v2
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../AuthorizationStmtTest.java | 9 +-
.../AuthorizationTest.java | 21 +-
.../TestSentryGroupMapper.java} | 8 +-
.../TestSentryResourceAuthorizationProvider.java} | 12 +-
fe/src/test/resources/sentry-site.xml.py | 2 +-
.../QueryTest/compute-stats-avro-catalog-v2.test | 378 +++++++++++++++++++++
.../queries/QueryTest/compute-stats-avro.test | 377 ++++++++++++++++++++
.../queries/QueryTest/compute-stats.test | 376 --------------------
.../queries/QueryTest/describe-db.test | 4 +-
tests/authorization/test_authorization.py | 4 +-
tests/authorization/test_grant_revoke.py | 4 +-
tests/authorization/test_owner_privileges.py | 16 +-
tests/authorization/test_show_grant.py | 8 +-
tests/catalog_service/test_hms_failure.py | 5 +-
tests/common/environ.py | 30 +-
tests/common/impala_test_suite.py | 14 +
tests/common/skip.py | 16 +
tests/metadata/test_compute_stats.py | 24 +-
tests/metadata/test_hms_integration.py | 8 +-
tests/metadata/test_metadata_query_statements.py | 11 +-
tests/metadata/test_refresh_partition.py | 7 +-
tests/query_test/test_avro_schema_resolution.py | 3 +-
tests/webserver/test_web_pages.py | 25 +-
23 files changed, 919 insertions(+), 443 deletions(-)
rename fe/src/test/java/org/apache/impala/{analysis => authorization}/AuthorizationStmtTest.java (99%)
rename fe/src/test/java/org/apache/impala/{analysis => authorization}/AuthorizationTest.java (97%)
rename fe/src/test/java/org/apache/impala/{service/CustomClusterGroupMapper.java => testutil/TestSentryGroupMapper.java} (93%)
rename fe/src/test/java/org/apache/impala/{service/CustomClusterResourceAuthorizationProvider.java => testutil/TestSentryResourceAuthorizationProvider.java} (74%)
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test
[impala] 02/02: IMPALA-8121: part 1: some test fixes for catalog v2
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 79c5f87565467074697a7d98e01c9742f7228991
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Thu Apr 25 11:13:16 2019 -0700
IMPALA-8121: part 1: some test fixes for catalog v2
This fixes some test issues encountered when running the
tests against a cluster with catalog V2 enabled, meaning
the local catalog with HMS notifications enabled. More
fixes are to come but I preferred to do them in smaller
batches as they're ready.
Test fixes:
* Detect whether catalog v2 features are enabled from web UI.
* test_describe_db waits for metadata event processor to pick up new
database and doesn't need to change database owner
* TestWebPage.test_catalog handles an expected exception from
the /catalog_objects page on the impalad.
* test_pull_stats_profile: feature disabled with local catalog
* test_hms_service_dies: invalidate the test table instead of
the whole catalog.
* test_compute_stats: Avro schema resolution behaviour changed
with local catalog - IMPALA-7308
Some remaining issues:
* IMPALA-8458
* IMPALA-8459
* IMPALA-7131 (data sources)
* getTables() doesn't return comment
Change-Id: I060f2076da74fbbe92ae26dbad51f09a3bd20169
Reviewed-on: http://gerrit.cloudera.org:8080/13122
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../QueryTest/compute-stats-avro-catalog-v2.test | 378 +++++++++++++++++++++
.../queries/QueryTest/compute-stats-avro.test | 377 ++++++++++++++++++++
.../queries/QueryTest/compute-stats.test | 376 --------------------
.../queries/QueryTest/describe-db.test | 4 +-
tests/catalog_service/test_hms_failure.py | 5 +-
tests/common/environ.py | 30 +-
tests/common/impala_test_suite.py | 14 +
tests/common/skip.py | 16 +
tests/metadata/test_compute_stats.py | 24 +-
tests/metadata/test_hms_integration.py | 8 +-
tests/metadata/test_metadata_query_statements.py | 11 +-
tests/metadata/test_refresh_partition.py | 7 +-
tests/query_test/test_avro_schema_resolution.py | 3 +-
tests/webserver/test_web_pages.py | 25 +-
14 files changed, 881 insertions(+), 397 deletions(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test
new file mode 100644
index 0000000..a411806
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test
@@ -0,0 +1,378 @@
+====
+---- QUERY
+# IMPALA-867: Test computing stats on Avro tables created by Hive with
+# matching/mismatched column definitions and Avro schema.
+# Clone the used tables here.
+create table avro_hive_alltypes
+like functional_avro_snap.alltypes;
+create table avro_hive_alltypes_extra_coldef
+like functional_avro_snap.alltypes_extra_coldef;
+create table avro_hive_alltypes_missing_coldef
+like functional_avro_snap.alltypes_missing_coldef;
+create table avro_hive_alltypes_type_mismatch
+like functional_avro_snap.alltypes_type_mismatch;
+create table avro_hive_no_avro_schema
+like functional_avro_snap.no_avro_schema;
+====
+---- QUERY
+# Avro table with matching column definitions and Avro schema
+compute stats avro_hive_alltypes
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with an extra column definition.
+compute stats avro_hive_alltypes_extra_coldef
+---- CATCH
+AnalysisException: Cannot COMPUTE STATS on Avro table 'avro_hive_alltypes_extra_coldef' because its column definitions do not match those in the Avro schema.
+Missing Avro-schema column corresponding to column definition 'extra_col' of type 'string' at position '10'.
+Please re-create the table with column definitions, e.g., using the result of 'SHOW CREATE TABLE'
+====
+---- QUERY
+show table stats avro_hive_alltypes_extra_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_extra_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',-1,-1,4,4
+'bool_col','BOOLEAN',-1,-1,1,1
+'tinyint_col','INT',-1,-1,4,4
+'smallint_col','INT',-1,-1,4,4
+'int_col','INT',-1,-1,4,4
+'bigint_col','BIGINT',-1,-1,8,8
+'float_col','FLOAT',-1,-1,4,4
+'double_col','DOUBLE',-1,-1,8,8
+'date_string_col','STRING',-1,-1,-1,-1
+'string_col','STRING',-1,-1,-1,-1
+'timestamp_col','STRING',-1,-1,-1,-1
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with missing two column definitions.
+compute stats avro_hive_alltypes_missing_coldef
+---- CATCH
+MESSAGE: AnalysisException: Cannot COMPUTE STATS on Avro table 'avro_hive_alltypes_missing_coldef' because its column definitions do not match those in the Avro schema.
+Definition of column 'smallint_col' of type 'smallint' does not match the Avro-schema column 'tinyint_col' of type 'INT' at position '2'.
+Please re-create the table with column definitions, e.g., using the result of 'SHOW CREATE TABLE'
+====
+---- QUERY
+show table stats avro_hive_alltypes_missing_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_missing_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'bigint_col','BIGINT',-1,-1,8,8
+'bool_col','BOOLEAN',-1,-1,1,1
+'date_string_col','STRING',-1,-1,-1,-1
+'double_col','DOUBLE',-1,-1,8,8
+'float_col','FLOAT',-1,-1,4,4
+'id','INT',-1,-1,4,4
+'int_col','INT',-1,-1,4,4
+'month','INT',0,0,4,4
+'smallint_col','INT',-1,-1,4,4
+'string_col','STRING',-1,-1,-1,-1
+'timestamp_col','STRING',-1,-1,-1,-1
+'tinyint_col','INT',-1,-1,4,4
+'year','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with one column definition having a different
+# type than the Avro schema (bigint_col is a string).
+compute stats avro_hive_alltypes_type_mismatch
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_type_mismatch
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_type_mismatch
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',-1,-1,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',-1,-1,-1,-1
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table without an Avro schema created by Hive.
+# The Avro schema is inferred from the column definitions,
+compute stats avro_hive_no_avro_schema
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_no_avro_schema
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_no_avro_schema
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test Avro table created without any column definitions.
+create table avro_impala_alltypes_no_coldefs
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_no_coldefs
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_no_coldefs
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_no_coldefs
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column name.
+create table avro_impala_alltypes_bad_colname
+(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
+bigint_col bigint, float_col float, double_col double, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_colname
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_colname
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_colname
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column type.
+create table avro_impala_alltypes_bad_coltype
+(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
+bigint_col bigint, float_col float, double_col bigint, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_coltype
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_coltype
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_coltype
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test that compute stats on a Hive-created Avro table without column defs
+# works (HIVE-6308, IMPALA-867).
+create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
+compute stats alltypes_no_coldef
+---- RESULTS
+'Updated 1 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show column stats alltypes_no_coldef
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test
new file mode 100644
index 0000000..6052d57
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test
@@ -0,0 +1,377 @@
+====
+---- QUERY
+# IMPALA-867: Test computing stats on Avro tables created by Hive with
+# matching/mismatched column definitions and Avro schema.
+# Clone the used tables here.
+create table avro_hive_alltypes
+like functional_avro_snap.alltypes;
+create table avro_hive_alltypes_extra_coldef
+like functional_avro_snap.alltypes_extra_coldef;
+create table avro_hive_alltypes_missing_coldef
+like functional_avro_snap.alltypes_missing_coldef;
+create table avro_hive_alltypes_type_mismatch
+like functional_avro_snap.alltypes_type_mismatch;
+create table avro_hive_no_avro_schema
+like functional_avro_snap.no_avro_schema;
+====
+---- QUERY
+# Avro table with matching column definitions and Avro schema
+compute stats avro_hive_alltypes
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with an extra column definition.
+compute stats avro_hive_alltypes_extra_coldef
+---- RESULTS
+'Updated 0 partition(s) and 12 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_extra_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_extra_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','TINYINT',0,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','TIMESTAMP',0,0,16,16
+'extra_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with missing two column definitions.
+compute stats avro_hive_alltypes_missing_coldef
+---- RESULTS
+'Updated 0 partition(s) and 9 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_missing_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_missing_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with one column definition having a different
+# type than the Avro schema (bigint_col is a string).
+compute stats avro_hive_alltypes_type_mismatch
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_type_mismatch
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_type_mismatch
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','TINYINT',0,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','STRING',0,0,0,0
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','TIMESTAMP',0,0,16,16
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table without an Avro schema created by Hive.
+# The Avro schema is inferred from the column definitions,
+compute stats avro_hive_no_avro_schema
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_no_avro_schema
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_no_avro_schema
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test Avro table created without any column definitions.
+create table avro_impala_alltypes_no_coldefs
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_no_coldefs
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_no_coldefs
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_no_coldefs
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column name.
+create table avro_impala_alltypes_bad_colname
+(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
+bigint_col bigint, float_col float, double_col double, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_colname
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_colname
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_colname
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column type.
+create table avro_impala_alltypes_bad_coltype
+(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
+bigint_col bigint, float_col float, double_col bigint, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_coltype
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_coltype
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_coltype
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test that compute stats on a Hive-created Avro table without column defs
+# works (HIVE-6308, IMPALA-867).
+create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
+compute stats alltypes_no_coldef
+---- RESULTS
+'Updated 1 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show column stats alltypes_no_coldef
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
index 9b17cd3..01e5aa8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
@@ -657,355 +657,6 @@ COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
====
---- QUERY
-# IMPALA-867: Test computing stats on Avro tables created by Hive with
-# matching/mismatched column definitions and Avro schema.
-# Clone the used tables here.
-create table avro_hive_alltypes
-like functional_avro_snap.alltypes;
-create table avro_hive_alltypes_extra_coldef
-like functional_avro_snap.alltypes_extra_coldef;
-create table avro_hive_alltypes_missing_coldef
-like functional_avro_snap.alltypes_missing_coldef;
-create table avro_hive_alltypes_type_mismatch
-like functional_avro_snap.alltypes_type_mismatch;
-create table avro_hive_no_avro_schema
-like functional_avro_snap.no_avro_schema;
-====
----- QUERY
-# Avro table with matching column definitions and Avro schema
-compute stats avro_hive_alltypes
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with an extra column definition.
-compute stats avro_hive_alltypes_extra_coldef
----- RESULTS
-'Updated 0 partition(s) and 12 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_extra_coldef
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_extra_coldef
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','TINYINT',0,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','TIMESTAMP',0,0,16,16
-'extra_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with missing two column definitions.
-compute stats avro_hive_alltypes_missing_coldef
----- RESULTS
-'Updated 0 partition(s) and 9 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_missing_coldef
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_missing_coldef
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with one column definition having a different
-# type than the Avro schema (bigint_col is a string).
-compute stats avro_hive_alltypes_type_mismatch
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_type_mismatch
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_type_mismatch
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','TINYINT',0,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','STRING',0,0,0,0
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','TIMESTAMP',0,0,16,16
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table without an Avro schema created by Hive.
-# The Avro schema is inferred from the column definitions,
-compute stats avro_hive_no_avro_schema
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_no_avro_schema
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_no_avro_schema
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Test Avro table created without any column definitions.
-create table avro_impala_alltypes_no_coldefs
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_no_coldefs
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_no_coldefs
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_no_coldefs
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# IMPALA-1104: Test computing stats on Avro tables created by Impala
-# with mismatched column definitions and Avro schema. Mismatched column name.
-create table avro_impala_alltypes_bad_colname
-(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
-bigint_col bigint, float_col float, double_col double, date_string_col string,
-string_col string, timestamp_col timestamp)
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_bad_colname
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_bad_colname
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_bad_colname
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# IMPALA-1104: Test computing stats on Avro tables created by Impala
-# with mismatched column definitions and Avro schema. Mismatched column type.
-create table avro_impala_alltypes_bad_coltype
-(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
-bigint_col bigint, float_col float, double_col bigint, date_string_col string,
-string_col string, timestamp_col timestamp)
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_bad_coltype
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_bad_coltype
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_bad_coltype
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
# IMPALA-883: Compute table stats for an empty partition.
create table empty_partitioned (i int) partitioned by (j int);
alter table empty_partitioned add partition (j=1);
@@ -1154,33 +805,6 @@ COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
====
---- QUERY
-# Test that compute stats on a Hive-created Avro table without column defs
-# works (HIVE-6308, IMPALA-867).
-create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
-compute stats alltypes_no_coldef
----- RESULTS
-'Updated 1 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show column stats alltypes_no_coldef
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
# Test that compute stats works on wide tables.
create table widetable_1000_cols
stored as parquet
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 027e19a..97159ab 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -21,7 +21,7 @@ describe database extended hive_test_desc_db
---- RESULTS
'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
'Parameter: ','',''
-'','test','USER'
+'','$USER','USER'
'Owner: ','',''
'','e','2.82'
'','pi','3.14'
@@ -72,4 +72,4 @@ describe database formatted impala_test_desc_db4
'impala_test_desc_db4','$NAMENODE/test2.db','test comment'
---- TYPES
string, string, string
-====
\ No newline at end of file
+====
diff --git a/tests/catalog_service/test_hms_failure.py b/tests/catalog_service/test_hms_failure.py
index d98f0b4..4e47b39 100644
--- a/tests/catalog_service/test_hms_failure.py
+++ b/tests/catalog_service/test_hms_failure.py
@@ -58,16 +58,15 @@ class TestHiveMetaStoreFailure(ImpalaTestSuite):
# Make sure the metastore is running even if the test aborts somewhere unexpected
# before restarting the metastore itself.
cls.run_hive_server()
- cls.client.execute("invalidate metadata")
super(TestHiveMetaStoreFailure, cls).teardown_class()
@pytest.mark.execute_serially
def test_hms_service_dies(self, vector):
"""Regression test for IMPALA-823 to verify the catalog service works properly when
HMS connections fail"""
- # Force all the tables to be reloaded and then kill the hive metastore.
+ # Force the tables to be uncached and then kill the hive metastore.
tbl_name = "functional.alltypes"
- self.client.execute("invalidate metadata")
+ self.client.execute("invalidate metadata %s" % tbl_name)
kill_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/kill-hive-server.sh')
check_call([kill_cmd], close_fds=True)
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 17b2329..f5df0a1 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -26,6 +26,9 @@ test_start_cluster_args = os.environ.get("TEST_START_CLUSTER_ARGS", "")
IMPALA_HOME = os.environ.get("IMPALA_HOME", "")
IMPALA_REMOTE_URL = os.environ.get("IMPALA_REMOTE_URL", "")
+# Default web UI URL for local test cluster
+DEFAULT_LOCAL_WEB_UI_URL = "http://localhost:25000"
+
# Find the likely BuildType of the running Impala. Assume it's found through the path
# $IMPALA_HOME/be/build/latest as a fallback.
build_type_arg_regex = re.compile(r'--build_type=(\w+)', re.I)
@@ -118,7 +121,7 @@ class ImpalaTestClusterFlagsDetector:
LOG.debug("Unable to read .cmake_build_type file, fetching build flags from " +
"web ui on localhost")
build_type, build_shared_libs = ImpalaTestClusterFlagsDetector.detect_using_web_ui(
- "http://localhost:25000")
+ DEFAULT_LOCAL_WEB_UI_URL)
library_link_type = LinkTypes.STATIC if build_shared_libs == "off"\
else LinkTypes.DYNAMIC
@@ -186,6 +189,7 @@ class ImpalaTestClusterProperties(object):
self._build_flavor = build_flavor
self._library_link_type = library_link_type
self._local_or_remote_build = local_or_remote_build
+ self._runtime_flags = None # Lazily populated to avoid unnecessary web UI calls.
@property
def build_flavor(self):
@@ -261,13 +265,37 @@ class ImpalaTestClusterProperties(object):
"""
return self._local_or_remote_build == REMOTE_BUILD
+ def _get_flags_from_web_ui(self, impala_url):
+ if self._runtime_flags is not None:
+ return self._runtime_flags
+ """Return the command line flags from the impala web UI. Returns a Python map with
+ the flag name as the key and a dictionary of flag properties as the value."""
+ response = requests.get(impala_url + "/varz?json")
+ assert response.status_code == requests.codes.ok,\
+ "Offending url: " + impala_url
+ assert "application/json" in response.headers['Content-Type']
+ self._runtime_flags = {}
+ for flag_dict in json.loads(response.text)["flags"]:
+ self._runtime_flags[flag_dict["name"]] = flag_dict
+ return self._runtime_flags
+
+ def is_catalog_v2_cluster(self):
+ """Whether we use CATALOG_V2 options, including local catalog and HMS notifications.
+ For now, assume that --use_local_catalog=true implies that the others are enabled."""
+ flags = self._get_flags_from_web_ui(web_ui_url)
+ key = "use_local_catalog"
+ # --use_local_catalog is hidden so does not appear in JSON if disabled.
+ return key in flags and flags[key]["current"] == "true"
+
if IMPALA_REMOTE_URL:
+ web_ui_url = IMPALA_REMOTE_URL
build_flavor, link_type =\
ImpalaTestClusterFlagsDetector.detect_using_web_ui(IMPALA_REMOTE_URL)
IMPALA_TEST_CLUSTER_PROPERTIES =\
ImpalaTestClusterProperties(build_flavor, link_type, REMOTE_BUILD)
else:
+ web_ui_url = DEFAULT_LOCAL_WEB_UI_URL
build_flavor, link_type =\
ImpalaTestClusterFlagsDetector.detect_using_build_root_or_web_ui(IMPALA_HOME)
IMPALA_TEST_CLUSTER_PROPERTIES =\
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 8b1f54f..7fe8216 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -883,6 +883,20 @@ class ImpalaTestSuite(BaseTestSuite):
actual_state))
return actual_state
+ def wait_for_db_to_appear(self, db_name, timeout_s):
+ """Wait until the database with 'db_name' is present in the impalad's local catalog.
+ Fail after timeout_s if the doesn't appear."""
+ start_time = time.time()
+ while time.time() - start_time < timeout_s:
+ try:
+ # This will throw an exception if the database is not present.
+ self.client.execute("describe database `{db_name}`".format(db_name=db_name))
+ return
+ except Exception:
+ time.sleep(0.2)
+ continue
+ raise Exception("Table {0} didn't show up after {1}s", db_name, timeout_s)
+
def assert_impalad_log_contains(self, level, line_regex, expected_count=1):
"""
Convenience wrapper around assert_log_contains for impalad logs.
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 95a62da..cad1caa 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -196,3 +196,19 @@ class SkipIfDockerizedCluster:
reason="IMPALA-4865: JVM hits OOM for large string. Heap is smaller in docker.")
insert_acls = pytest.mark.skipif(IS_DOCKERIZED_TEST_CLUSTER,
reason="IMPALA-8384: insert ACL tests are broken on dockerised minicluster.")
+
+
+class SkipIfCatalogV2:
+ """Expose decorators as methods so that is_catalog_v2_cluster() can be evaluated lazily
+ when needed, instead of whenever this module is imported."""
+ @classmethod
+ def stats_pulling_disabled(self):
+ return pytest.mark.skipif(
+ IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
+ reason="Local catalog does not use incremental stats pulling.")
+
+ @classmethod
+ def catalog_v1_test(self):
+ return pytest.mark.skipif(
+ IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
+ reason="Test is specific to old implementation of catalog.")
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index 5673438..7ae05f6 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -18,9 +18,11 @@
import pytest
from subprocess import check_call
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
+ SkipIfLocal, SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_single_exec_option_dimension,
@@ -47,9 +49,28 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfS3.eventually_consistent
def test_compute_stats(self, vector, unique_database):
self.run_test_case('QueryTest/compute-stats', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_avro(self, vector, unique_database):
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # IMPALA-7308: changed behaviour of various Avro edge cases significantly in the
+ # local catalog - the expected behaviour is different.
+ self.run_test_case('QueryTest/compute-stats-avro-catalog-v2', vector,
+ unique_database)
+ else:
+ self.run_test_case('QueryTest/compute-stats-avro', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_decimal(self, vector, unique_database):
# Test compute stats on decimal columns separately so we can vary between platforms
# with and without write support for decimals (Hive < 0.11 and >= 0.11).
self.run_test_case('QueryTest/compute-stats-decimal', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_date(self, vector, unique_database):
# Test compute stats on date columns separately.
self.run_test_case('QueryTest/compute-stats-date', vector, unique_database)
@@ -142,6 +163,7 @@ class TestComputeStats(ImpalaTestSuite):
assert("1\tpval\t8" in show_result.data[0])
@SkipIfS3.eventually_consistent
+ @SkipIfCatalogV2.stats_pulling_disabled()
def test_pull_stats_profile(self, vector, unique_database):
"""Checks that the frontend profile includes metrics when computing
incremental statistics.
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 800799c..71a0879 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -29,6 +29,7 @@ import random
import string
from subprocess import call
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (
@@ -62,7 +63,12 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
self.run_stmt_in_hive("drop database if exists hms_sanity_db cascade")
self.run_stmt_in_hive("create database hms_sanity_db")
# Make sure Impala's metadata is in sync.
- self.client.execute("invalidate metadata")
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # Using local catalog + HMS event processor - wait until the database shows up.
+ self.wait_for_db_to_appear("hms_sanity_db", timeout_s=30)
+ else:
+ # Using traditional catalog - need to invalidate to pick up hive-created db.
+ self.client.execute("invalidate metadata")
# Creating a database with the same name using 'IF NOT EXISTS' in Impala should
# not fail
self.client.execute("create database if not exists hms_sanity_db")
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 81fd60d..5633ba0 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -21,6 +21,7 @@ import pytest
import re
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfIsilon, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import ALL_NODES_ONLY
@@ -154,7 +155,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
- @pytest.mark.execute_serially # because of invalidate metadata
+ @pytest.mark.execute_serially # because of use of hardcoded database
def test_describe_db(self, vector):
self.__test_describe_db_cleanup()
try:
@@ -167,8 +168,12 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
"location \"" + get_fs_path("/test2.db") + "\"")
self.run_stmt_in_hive("create database hive_test_desc_db comment 'test comment' "
"with dbproperties('pi' = '3.14', 'e' = '2.82')")
- self.run_stmt_in_hive("alter database hive_test_desc_db set owner user test")
- self.client.execute("invalidate metadata")
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # Using local catalog + HMS event processor - wait until the database shows up.
+ self.wait_for_db_to_appear("hive_test_desc_db", timeout_s=30)
+ else:
+ # Using traditional catalog - need to invalidate to pick up hive-created db.
+ self.client.execute("invalidate metadata")
self.run_test_case('QueryTest/describe-db', vector)
finally:
self.__test_describe_db_cleanup()
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index 4b2c67b..6d9c058 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -14,6 +14,7 @@
from subprocess import check_call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -133,7 +134,11 @@ class TestRefreshPartition(ImpalaTestSuite):
% table_name)
# Make sure its still shows the same result before refreshing
result = self.client.execute("select count(*) from %s" % table_name)
- assert result.data == [str('0')]
+ valid_counts = [0]
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # HMS notifications may pick up added partition racily.
+ valid_counts.append(1)
+ assert int(result.data[0]) in valid_counts
self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
assert '2\t333\t5309' == self.client.execute(
diff --git a/tests/query_test/test_avro_schema_resolution.py b/tests/query_test/test_avro_schema_resolution.py
index e660101..52feb8a 100644
--- a/tests/query_test/test_avro_schema_resolution.py
+++ b/tests/query_test/test_avro_schema_resolution.py
@@ -16,6 +16,7 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfCatalogV2
# This test requires that testdata/avro_schema_resolution/create_table.sql has been run
class TestAvroSchemaResolution(ImpalaTestSuite):
@@ -47,6 +48,7 @@ class TestAvroSchemaResolution(ImpalaTestSuite):
for x in range(len(result.data)):
assert comparison.data[x] == result.data[x]
+ @SkipIfCatalogV2.catalog_v1_test()
def test_avro_schema_changes(self, vector, unique_database):
"""Test for IMPALA-3314 and IMPALA-3513: Impalad shouldn't crash with stale Avro
metadata. Instead, should provide a meaningful error message.
@@ -54,5 +56,4 @@ class TestAvroSchemaResolution(ImpalaTestSuite):
... ADD COLUMN ...
Test for IMPALA-3776: Fix describe formatted when changing Avro schema.
"""
- # TODO(todd): skip the "stale metadata" tests if LocalCatalog is enabled
self.run_test_case('QueryTest/avro-schema-changes', vector, unique_database)
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index cb165d4..2960599 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
-from tests.common.environ import ImpalaTestClusterFlagsDetector
+from tests.common.environ import (IMPALA_TEST_CLUSTER_PROPERTIES,
+ ImpalaTestClusterFlagsDetector)
from tests.common.skip import SkipIfBuildType
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -53,6 +53,7 @@ class TestWebPage(ImpalaTestSuite):
# one with it.
TEST_PORTS_WITHOUT_SS = ["25000", "25020"]
TEST_PORTS_WITH_SS = ["25000", "25010", "25020"]
+ IMPALAD_TEST_PORT = ["25000"]
CATALOG_TEST_PORT = ["25020"]
def test_get_root_url(self):
@@ -287,14 +288,22 @@ class TestWebPage(ImpalaTestSuite):
def __test_catalog_object(self, db_name, tbl_name):
"""Tests the /catalog_object endpoint for the given db/table. Runs
against an unloaded as well as a loaded table."""
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ impalad_expected_str = \
+ "UnsupportedOperationException: LocalCatalog.getTCatalogObject"
+ else:
+ impalad_expected_str = tbl_name
+ obj_url = self.CATALOG_OBJECT_URL + \
+ "?object_type=TABLE&object_name={0}.{1}".format(db_name, tbl_name)
self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name))
- self.get_and_check_status(self.CATALOG_OBJECT_URL +
- "?object_type=TABLE&object_name=%s.%s" % (db_name, tbl_name), tbl_name,
- ports_to_test=self.TEST_PORTS_WITHOUT_SS)
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
self.client.execute("select count(*) from %s.%s" % (db_name, tbl_name))
- self.get_and_check_status(self.CATALOG_OBJECT_URL +
- "?object_type=TABLE&object_name=%s.%s" % (db_name, tbl_name), tbl_name,
- ports_to_test=self.TEST_PORTS_WITHOUT_SS)
+
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
def __test_table_metrics(self, db_name, tbl_name, metric):
self.client.execute("refresh %s.%s" % (db_name, tbl_name))
[impala] 01/02: IMPALA-8269: Refactor authorization test package
structure
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d29d68c90d557353f7e312ba99747a6adca23820
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Wed May 1 10:28:17 2019 -0700
IMPALA-8269: Refactor authorization test package structure
This patch refactors authorization test package structure by moving it
to the authorization test package. This patch also renames
CustomClusterResourceAuthorizationProvider to
TestSentryResourceAuthorizationProvider since it is a class that is
specific to Sentry and is only used for testing.
Testing:
- Ran all FE tests
- Ran all E2E authorization tests
Change-Id: I525ff71f63d7c306d82b4c111f98ff327e4a07b3
Reviewed-on: http://gerrit.cloudera.org:8080/13208
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../AuthorizationStmtTest.java | 9 ++-------
.../AuthorizationTest.java | 21 +++++++++------------
.../TestSentryGroupMapper.java} | 8 ++++----
.../TestSentryResourceAuthorizationProvider.java} | 12 ++++++------
fe/src/test/resources/sentry-site.xml.py | 2 +-
tests/authorization/test_authorization.py | 4 ++--
tests/authorization/test_grant_revoke.py | 4 ++--
tests/authorization/test_owner_privileges.py | 16 ++++++++--------
tests/authorization/test_show_grant.py | 8 ++++----
9 files changed, 38 insertions(+), 46 deletions(-)
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
similarity index 99%
rename from fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
rename to fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index df543cf..b151a56 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.impala.analysis;
+package org.apache.impala.authorization;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -33,13 +33,8 @@ import java.util.stream.Collectors;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.impala.analysis.AnalysisContext;
import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
-import org.apache.impala.authorization.AuthorizationConfig;
-import org.apache.impala.authorization.AuthorizationFactory;
-import org.apache.impala.authorization.AuthorizationProvider;
-import org.apache.impala.authorization.PrivilegeRequest;
-import org.apache.impala.authorization.User;
-import org.apache.impala.authorization.AuthorizationException;
import org.apache.impala.authorization.ranger.RangerAuthorizationChecker;
import org.apache.impala.authorization.ranger.RangerAuthorizationConfig;
import org.apache.impala.authorization.ranger.RangerAuthorizationFactory;
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTest.java
similarity index 97%
rename from fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
rename to fe/src/test/java/org/apache/impala/authorization/AuthorizationTest.java
index e319bf3..b8ab93b 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTest.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.impala.analysis;
+package org.apache.impala.authorization;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
import static org.junit.Assert.assertEquals;
@@ -32,11 +32,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
-import org.apache.impala.authorization.AuthorizationConfig;
-import org.apache.impala.authorization.AuthorizationFactory;
-import org.apache.impala.authorization.User;
+import org.apache.impala.analysis.AnalysisContext;
import org.apache.impala.authorization.sentry.SentryAuthorizationConfig;
-import org.apache.impala.authorization.AuthorizationException;
import org.apache.impala.authorization.sentry.SentryAuthorizationFactory;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.FeDb;
@@ -46,8 +43,8 @@ import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.RuntimeEnv;
-import org.apache.impala.service.CustomClusterGroupMapper;
-import org.apache.impala.service.CustomClusterResourceAuthorizationProvider;
+import org.apache.impala.testutil.TestSentryGroupMapper;
+import org.apache.impala.testutil.TestSentryResourceAuthorizationProvider;
import org.apache.impala.service.Frontend;
import org.apache.impala.testutil.ImpaladTestCatalog;
import org.apache.impala.thrift.TMetadataOpRequest;
@@ -122,13 +119,13 @@ public class AuthorizationTest extends FrontendTestBase {
false);
Role role = catalog.addRole(role_);
addRolePrivilege(catalog, privilege, role);
- catalog.addRoleGrantGroup(role_, CustomClusterGroupMapper.SERVER_ADMIN);
+ catalog.addRoleGrantGroup(role_, TestSentryGroupMapper.SERVER_ADMIN);
// Create test users
Set<String> groups = new HashSet<>(Arrays.asList(
USER.getName(),
- CustomClusterGroupMapper.AUTH_TO_LOCAL,
- CustomClusterGroupMapper.TEST_USER));
+ TestSentryGroupMapper.AUTH_TO_LOCAL,
+ TestSentryGroupMapper.TEST_USER));
role_ = "testRole";
role = catalog.addRole(role_);
@@ -470,7 +467,7 @@ public class AuthorizationTest extends FrontendTestBase {
// </property>
SentryAuthorizationConfig authzConfig = new SentryAuthorizationConfig("server1",
AuthorizationTest.AUTHZ_CONFIG.getSentryConfig().getConfigFile(),
- CustomClusterResourceAuthorizationProvider.class.getName());
+ TestSentryResourceAuthorizationProvider.class.getName());
SentryAuthorizationFactory authzFactory = new SentryAuthorizationFactory(
authzConfig);
try (ImpaladTestCatalog catalog = new ImpaladTestCatalog(authzFactory)) {
@@ -625,7 +622,7 @@ public class AuthorizationTest extends FrontendTestBase {
// CustomClusterResourceAuthorizationProvider.
SentryAuthorizationConfig authzConfig = new SentryAuthorizationConfig("server1",
AuthorizationTest.AUTHZ_CONFIG.getSentryConfig().getConfigFile(),
- CustomClusterResourceAuthorizationProvider.class.getName());
+ TestSentryResourceAuthorizationProvider.class.getName());
SentryAuthorizationFactory authzFactory = new SentryAuthorizationFactory(authzConfig);
try (ImpaladTestCatalog catalog = new ImpaladTestCatalog(authzFactory)) {
setupImpalaCatalog(catalog);
diff --git a/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java b/fe/src/test/java/org/apache/impala/testutil/TestSentryGroupMapper.java
similarity index 93%
rename from fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
rename to fe/src/test/java/org/apache/impala/testutil/TestSentryGroupMapper.java
index 5cb34fb..8ed8a7e 100644
--- a/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestSentryGroupMapper.java
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.impala.service;
+package org.apache.impala.testutil;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -30,7 +30,7 @@ import java.util.Set;
* This class is used for testing complex privileges where we don't create
* users and groups on the system.
*/
-public class CustomClusterGroupMapper implements GroupMappingService {
+public class TestSentryGroupMapper implements GroupMappingService {
private final Map<String, Set<String>> groupsMap_ = Maps.newHashMap();
public static final String SERVER_ADMIN = "server_admin";
@@ -38,11 +38,11 @@ public class CustomClusterGroupMapper implements GroupMappingService {
public static final String AUTH_TO_LOCAL = "auth_to_local_group";
// Needed for sentry service to lookup groups.
- public CustomClusterGroupMapper(Configuration conf, String resource) {
+ public TestSentryGroupMapper(Configuration conf, String resource) {
this();
}
- public CustomClusterGroupMapper() {
+ public TestSentryGroupMapper() {
// Need to make sure we can resolve the dev user.
String devUser = System.getProperty("user.name");
groupsMap_.put(devUser, Sets.newHashSet(devUser));
diff --git a/fe/src/test/java/org/apache/impala/service/CustomClusterResourceAuthorizationProvider.java b/fe/src/test/java/org/apache/impala/testutil/TestSentryResourceAuthorizationProvider.java
similarity index 74%
rename from fe/src/test/java/org/apache/impala/service/CustomClusterResourceAuthorizationProvider.java
rename to fe/src/test/java/org/apache/impala/testutil/TestSentryResourceAuthorizationProvider.java
index 3c7d77f..a1097bb 100644
--- a/fe/src/test/java/org/apache/impala/service/CustomClusterResourceAuthorizationProvider.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestSentryResourceAuthorizationProvider.java
@@ -15,22 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.impala.service;
+package org.apache.impala.testutil;
import org.apache.hadoop.conf.Configuration;
import org.apache.sentry.core.common.Model;
import org.apache.sentry.policy.common.PolicyEngine;
import org.apache.sentry.provider.common.ResourceAuthorizationProvider;
-public class CustomClusterResourceAuthorizationProvider
+public class TestSentryResourceAuthorizationProvider
extends ResourceAuthorizationProvider {
- public CustomClusterResourceAuthorizationProvider(String resource, PolicyEngine policy,
+ public TestSentryResourceAuthorizationProvider(String resource, PolicyEngine policy,
Model model) {
- super(policy, new CustomClusterGroupMapper(), model);
+ super(policy, new TestSentryGroupMapper(), model);
}
- public CustomClusterResourceAuthorizationProvider(Configuration conf, String resource,
+ public TestSentryResourceAuthorizationProvider(Configuration conf, String resource,
PolicyEngine policy, Model model) {
- super(policy, new CustomClusterGroupMapper(), model);
+ super(policy, new TestSentryGroupMapper(), model);
}
}
diff --git a/fe/src/test/resources/sentry-site.xml.py b/fe/src/test/resources/sentry-site.xml.py
index 64a25fd..9225622 100644
--- a/fe/src/test/resources/sentry-site.xml.py
+++ b/fe/src/test/resources/sentry-site.xml.py
@@ -58,7 +58,7 @@ if variant is not None:
'sentry.hive.testing.mode': 'true',
'sentry.policy.store.plugins': 'org.apache.sentry.hdfs.SentryPlugin',
# Custom group mapping for custom cluster tests .
- 'sentry.store.group.mapping': 'org.apache.impala.service.CustomClusterGroupMapper',
+ 'sentry.store.group.mapping': 'org.apache.impala.testutil.TestSentryGroupMapper',
'sentry.service.processor.factories': '${SENTRY_PROCESSOR_FACTORIES}',
})
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index 4f309e8..a7e7361 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -103,11 +103,11 @@ class TestAuthorization(CustomClusterTestSuite):
impalad_args="--server_name=server1 "
"--sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE),
catalogd_args="--sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE),
sentry_config=SENTRY_CONFIG_FILE)
def test_custom_authorization_provider(self, unique_role):
diff --git a/tests/authorization/test_grant_revoke.py b/tests/authorization/test_grant_revoke.py
index 542f749..f68f6fb 100644
--- a/tests/authorization/test_grant_revoke.py
+++ b/tests/authorization/test_grant_revoke.py
@@ -340,11 +340,11 @@ class TestGrantRevoke(SentryCacheTestSuite):
@SentryCacheTestSuite.with_args(
impalad_args="--server_name=server1 --sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO),
catalogd_args="--sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO),
sentry_config=SENTRY_CONFIG_FILE_OO)
def test_same_name_for_role_and_user_invalidate_metadata(self, unique_name):
diff --git a/tests/authorization/test_owner_privileges.py b/tests/authorization/test_owner_privileges.py
index 38bb8f7..ea0e4ac 100644
--- a/tests/authorization/test_owner_privileges.py
+++ b/tests/authorization/test_owner_privileges.py
@@ -118,11 +118,11 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
@SentryCacheTestSuite.with_args(
impalad_args="--server_name=server1 --sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO),
catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO, SENTRY_LONG_POLLING_FREQUENCY_S),
sentry_config=SENTRY_CONFIG_FILE_OO,
sentry_log_dir="{0}/test_owner_privileges_with_grant".format(SENTRY_BASE_LOG_DIR))
@@ -236,10 +236,10 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
@SentryCacheTestSuite.with_args(
impalad_args="--server_name=server1 --sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider "
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_NO_OO),
catalogd_args="--sentry_config={0} --authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_NO_OO),
sentry_config=SENTRY_CONFIG_FILE_NO_OO,
sentry_log_dir="{0}/test_owner_privileges_disabled".format(SENTRY_BASE_LOG_DIR))
@@ -295,11 +295,11 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
@SentryCacheTestSuite.with_args(
impalad_args="--server_name=server1 --sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO_NOGRANT),
catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO_NOGRANT,
SENTRY_LONG_POLLING_FREQUENCY_S),
sentry_config=SENTRY_CONFIG_FILE_OO_NOGRANT,
@@ -368,11 +368,11 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
@SentryCacheTestSuite.with_args(
impalad_args="--server_name=server1 --sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO),
catalogd_args="--sentry_config={0} "
"--authorization_policy_provider_class="
- "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ "org.apache.impala.testutil.TestSentryResourceAuthorizationProvider"
.format(SENTRY_CONFIG_FILE_OO),
sentry_config=SENTRY_CONFIG_FILE_OO,
sentry_log_dir="{0}/test_owner_privileges_different_cases"
diff --git a/tests/authorization/test_show_grant.py b/tests/authorization/test_show_grant.py
index cf2e543..0cbfc48 100644
--- a/tests/authorization/test_show_grant.py
+++ b/tests/authorization/test_show_grant.py
@@ -81,10 +81,10 @@ class TestShowGrant(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args='--server_name=server1 --authorization_policy_provider_class='
- 'org.apache.impala.service.CustomClusterResourceAuthorizationProvider '
+ 'org.apache.impala.testutil.TestSentryResourceAuthorizationProvider '
'--sentry_config={0}'.format(SENTRY_CONFIG_FILE),
catalogd_args='--sentry_config={0} --authorization_policy_provider_class='
- 'org.apache.impala.service.CustomClusterResourceAuthorizationProvider'
+ 'org.apache.impala.testutil.TestSentryResourceAuthorizationProvider'
.format(SENTRY_CONFIG_FILE),
sentry_config=SENTRY_CONFIG_FILE)
def test_show_grant_user(self, vector, unique_database):
@@ -97,10 +97,10 @@ class TestShowGrant(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args='--server_name=server1 --authorization_policy_provider_class='
- 'org.apache.impala.service.CustomClusterResourceAuthorizationProvider '
+ 'org.apache.impala.testutil.TestSentryResourceAuthorizationProvider '
'--sentry_config={0}'.format(SENTRY_CONFIG_FILE),
catalogd_args='--sentry_config={0} --authorization_policy_provider_class='
- 'org.apache.impala.service.CustomClusterResourceAuthorizationProvider'
+ 'org.apache.impala.testutil.TestSentryResourceAuthorizationProvider'
.format(SENTRY_CONFIG_FILE),
sentry_config=SENTRY_CONFIG_FILE)
def test_show_grant_in_hs2(self, vector, unique_database):