You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/05/02 23:33:52 UTC
[impala] 02/02: IMPALA-8121: part 1: some test fixes for catalog v2
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 79c5f87565467074697a7d98e01c9742f7228991
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Thu Apr 25 11:13:16 2019 -0700
IMPALA-8121: part 1: some test fixes for catalog v2
This fixes some test issues encountered when running the
tests against a cluster with catalog V2 enabled, meaning
the local catalog with HMS notifications enabled. More
fixes are to come but I preferred to do them in smaller
batches as they're ready.
Test fixes:
* Detect whether catalog v2 features are enabled from web UI.
* test_describe_db waits for metadata event processor to pick up new
database and doesn't need to change database owner
* TestWebPage.test_catalog handles an expected exception from
the /catalog_objects page on the impalad.
* test_pull_stats_profile: feature disabled with local catalog
* test_hms_service_dies: invalidate the test table instead of
the whole catalog.
* test_compute_stats: Avro schema resolution behaviour changed
with local catalog - IMPALA-7308
Some remaining issues:
* IMPALA-8458
* IMPALA-8459
* IMPALA-7131 (data sources)
* getTables() doesn't return comment
Change-Id: I060f2076da74fbbe92ae26dbad51f09a3bd20169
Reviewed-on: http://gerrit.cloudera.org:8080/13122
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../QueryTest/compute-stats-avro-catalog-v2.test | 378 +++++++++++++++++++++
.../queries/QueryTest/compute-stats-avro.test | 377 ++++++++++++++++++++
.../queries/QueryTest/compute-stats.test | 376 --------------------
.../queries/QueryTest/describe-db.test | 4 +-
tests/catalog_service/test_hms_failure.py | 5 +-
tests/common/environ.py | 30 +-
tests/common/impala_test_suite.py | 14 +
tests/common/skip.py | 16 +
tests/metadata/test_compute_stats.py | 24 +-
tests/metadata/test_hms_integration.py | 8 +-
tests/metadata/test_metadata_query_statements.py | 11 +-
tests/metadata/test_refresh_partition.py | 7 +-
tests/query_test/test_avro_schema_resolution.py | 3 +-
tests/webserver/test_web_pages.py | 25 +-
14 files changed, 881 insertions(+), 397 deletions(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test
new file mode 100644
index 0000000..a411806
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro-catalog-v2.test
@@ -0,0 +1,378 @@
+====
+---- QUERY
+# IMPALA-867: Test computing stats on Avro tables created by Hive with
+# matching/mismatched column definitions and Avro schema.
+# Clone the used tables here.
+create table avro_hive_alltypes
+like functional_avro_snap.alltypes;
+create table avro_hive_alltypes_extra_coldef
+like functional_avro_snap.alltypes_extra_coldef;
+create table avro_hive_alltypes_missing_coldef
+like functional_avro_snap.alltypes_missing_coldef;
+create table avro_hive_alltypes_type_mismatch
+like functional_avro_snap.alltypes_type_mismatch;
+create table avro_hive_no_avro_schema
+like functional_avro_snap.no_avro_schema;
+====
+---- QUERY
+# Avro table with matching column definitions and Avro schema
+compute stats avro_hive_alltypes
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with an extra column definition.
+compute stats avro_hive_alltypes_extra_coldef
+---- CATCH
+AnalysisException: Cannot COMPUTE STATS on Avro table 'avro_hive_alltypes_extra_coldef' because its column definitions do not match those in the Avro schema.
+Missing Avro-schema column corresponding to column definition 'extra_col' of type 'string' at position '10'.
+Please re-create the table with column definitions, e.g., using the result of 'SHOW CREATE TABLE'
+====
+---- QUERY
+show table stats avro_hive_alltypes_extra_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_extra_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',-1,-1,4,4
+'bool_col','BOOLEAN',-1,-1,1,1
+'tinyint_col','INT',-1,-1,4,4
+'smallint_col','INT',-1,-1,4,4
+'int_col','INT',-1,-1,4,4
+'bigint_col','BIGINT',-1,-1,8,8
+'float_col','FLOAT',-1,-1,4,4
+'double_col','DOUBLE',-1,-1,8,8
+'date_string_col','STRING',-1,-1,-1,-1
+'string_col','STRING',-1,-1,-1,-1
+'timestamp_col','STRING',-1,-1,-1,-1
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with missing two column definitions.
+compute stats avro_hive_alltypes_missing_coldef
+---- CATCH
+MESSAGE: AnalysisException: Cannot COMPUTE STATS on Avro table 'avro_hive_alltypes_missing_coldef' because its column definitions do not match those in the Avro schema.
+Definition of column 'smallint_col' of type 'smallint' does not match the Avro-schema column 'tinyint_col' of type 'INT' at position '2'.
+Please re-create the table with column definitions, e.g., using the result of 'SHOW CREATE TABLE'
+====
+---- QUERY
+show table stats avro_hive_alltypes_missing_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',-1,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_missing_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'bigint_col','BIGINT',-1,-1,8,8
+'bool_col','BOOLEAN',-1,-1,1,1
+'date_string_col','STRING',-1,-1,-1,-1
+'double_col','DOUBLE',-1,-1,8,8
+'float_col','FLOAT',-1,-1,4,4
+'id','INT',-1,-1,4,4
+'int_col','INT',-1,-1,4,4
+'month','INT',0,0,4,4
+'smallint_col','INT',-1,-1,4,4
+'string_col','STRING',-1,-1,-1,-1
+'timestamp_col','STRING',-1,-1,-1,-1
+'tinyint_col','INT',-1,-1,4,4
+'year','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with one column definition having a different
+# type than the Avro schema (bigint_col is a string).
+compute stats avro_hive_alltypes_type_mismatch
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_type_mismatch
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_type_mismatch
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',-1,-1,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',-1,-1,-1,-1
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table without an Avro schema created by Hive.
+# The Avro schema is inferred from the column definitions,
+compute stats avro_hive_no_avro_schema
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_no_avro_schema
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_no_avro_schema
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test Avro table created without any column definitions.
+create table avro_impala_alltypes_no_coldefs
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_no_coldefs
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_no_coldefs
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_no_coldefs
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column name.
+create table avro_impala_alltypes_bad_colname
+(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
+bigint_col bigint, float_col float, double_col double, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_colname
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_colname
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_colname
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column type.
+create table avro_impala_alltypes_bad_coltype
+(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
+bigint_col bigint, float_col float, double_col bigint, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_coltype
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_coltype
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_coltype
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test that compute stats on a Hive-created Avro table without column defs
+# works (HIVE-6308, IMPALA-867).
+create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
+compute stats alltypes_no_coldef
+---- RESULTS
+'Updated 1 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show column stats alltypes_no_coldef
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test
new file mode 100644
index 0000000..6052d57
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats-avro.test
@@ -0,0 +1,377 @@
+====
+---- QUERY
+# IMPALA-867: Test computing stats on Avro tables created by Hive with
+# matching/mismatched column definitions and Avro schema.
+# Clone the used tables here.
+create table avro_hive_alltypes
+like functional_avro_snap.alltypes;
+create table avro_hive_alltypes_extra_coldef
+like functional_avro_snap.alltypes_extra_coldef;
+create table avro_hive_alltypes_missing_coldef
+like functional_avro_snap.alltypes_missing_coldef;
+create table avro_hive_alltypes_type_mismatch
+like functional_avro_snap.alltypes_type_mismatch;
+create table avro_hive_no_avro_schema
+like functional_avro_snap.no_avro_schema;
+====
+---- QUERY
+# Avro table with matching column definitions and Avro schema
+compute stats avro_hive_alltypes
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with an extra column definition.
+compute stats avro_hive_alltypes_extra_coldef
+---- RESULTS
+'Updated 0 partition(s) and 12 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_extra_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_extra_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','TINYINT',0,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','TIMESTAMP',0,0,16,16
+'extra_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with missing two column definitions.
+compute stats avro_hive_alltypes_missing_coldef
+---- RESULTS
+'Updated 0 partition(s) and 9 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_missing_coldef
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_missing_coldef
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table with one column definition having a different
+# type than the Avro schema (bigint_col is a string).
+compute stats avro_hive_alltypes_type_mismatch
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_alltypes_type_mismatch
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_alltypes_type_mismatch
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','TINYINT',0,0,1,1
+'smallint_col','SMALLINT',0,0,2,2
+'int_col','INT',0,0,4,4
+'bigint_col','STRING',0,0,0,0
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','TIMESTAMP',0,0,16,16
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Avro table without an Avro schema created by Hive.
+# The Avro schema is inferred from the column definitions,
+compute stats avro_hive_no_avro_schema
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_hive_no_avro_schema
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_hive_no_avro_schema
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test Avro table created without any column definitions.
+create table avro_impala_alltypes_no_coldefs
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_no_coldefs
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_no_coldefs
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_no_coldefs
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column name.
+create table avro_impala_alltypes_bad_colname
+(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
+bigint_col bigint, float_col float, double_col double, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_colname
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_colname
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_colname
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# IMPALA-1104: Test computing stats on Avro tables created by Impala
+# with mismatched column definitions and Avro schema. Mismatched column type.
+create table avro_impala_alltypes_bad_coltype
+(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
+bigint_col bigint, float_col float, double_col bigint, date_string_col string,
+string_col string, timestamp_col timestamp)
+partitioned by (year int, month int)
+with serdeproperties
+('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
+stored as avro;
+====
+---- QUERY
+compute stats avro_impala_alltypes_bad_coltype
+---- RESULTS
+'Updated 0 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show table stats avro_impala_alltypes_bad_coltype
+---- LABELS
+YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
+---- RESULTS
+'Total','',0,0,'0B','0B','','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show column stats avro_impala_alltypes_bad_coltype
+---- LABELS
+COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+'year','INT',0,0,4,4
+'month','INT',0,0,4,4
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
+---- QUERY
+# Test that compute stats on a Hive-created Avro table without column defs
+# works (HIVE-6308, IMPALA-867).
+create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
+compute stats alltypes_no_coldef
+---- RESULTS
+'Updated 1 partition(s) and 11 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show column stats alltypes_no_coldef
+---- RESULTS
+'id','INT',0,0,4,4
+'bool_col','BOOLEAN',2,0,1,1
+'tinyint_col','INT',0,0,4,4
+'smallint_col','INT',0,0,4,4
+'int_col','INT',0,0,4,4
+'bigint_col','BIGINT',0,0,8,8
+'float_col','FLOAT',0,0,4,4
+'double_col','DOUBLE',0,0,8,8
+'date_string_col','STRING',0,0,0,0
+'string_col','STRING',0,0,0,0
+'timestamp_col','STRING',0,0,0,0
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
index 9b17cd3..01e5aa8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
@@ -657,355 +657,6 @@ COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
====
---- QUERY
-# IMPALA-867: Test computing stats on Avro tables created by Hive with
-# matching/mismatched column definitions and Avro schema.
-# Clone the used tables here.
-create table avro_hive_alltypes
-like functional_avro_snap.alltypes;
-create table avro_hive_alltypes_extra_coldef
-like functional_avro_snap.alltypes_extra_coldef;
-create table avro_hive_alltypes_missing_coldef
-like functional_avro_snap.alltypes_missing_coldef;
-create table avro_hive_alltypes_type_mismatch
-like functional_avro_snap.alltypes_type_mismatch;
-create table avro_hive_no_avro_schema
-like functional_avro_snap.no_avro_schema;
-====
----- QUERY
-# Avro table with matching column definitions and Avro schema
-compute stats avro_hive_alltypes
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with an extra column definition.
-compute stats avro_hive_alltypes_extra_coldef
----- RESULTS
-'Updated 0 partition(s) and 12 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_extra_coldef
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_extra_coldef
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','TINYINT',0,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','TIMESTAMP',0,0,16,16
-'extra_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with missing two column definitions.
-compute stats avro_hive_alltypes_missing_coldef
----- RESULTS
-'Updated 0 partition(s) and 9 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_missing_coldef
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_missing_coldef
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table with one column definition having a different
-# type than the Avro schema (bigint_col is a string).
-compute stats avro_hive_alltypes_type_mismatch
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_alltypes_type_mismatch
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_alltypes_type_mismatch
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','TINYINT',0,0,1,1
-'smallint_col','SMALLINT',0,0,2,2
-'int_col','INT',0,0,4,4
-'bigint_col','STRING',0,0,0,0
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','TIMESTAMP',0,0,16,16
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Avro table without an Avro schema created by Hive.
-# The Avro schema is inferred from the column definitions,
-compute stats avro_hive_no_avro_schema
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_hive_no_avro_schema
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_hive_no_avro_schema
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# Test Avro table created without any column definitions.
-create table avro_impala_alltypes_no_coldefs
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_no_coldefs
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_no_coldefs
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_no_coldefs
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# IMPALA-1104: Test computing stats on Avro tables created by Impala
-# with mismatched column definitions and Avro schema. Mismatched column name.
-create table avro_impala_alltypes_bad_colname
-(id int, bool_col boolean, tinyint_col int, smallint_col int, bad_int_col int,
-bigint_col bigint, float_col float, double_col double, date_string_col string,
-string_col string, timestamp_col timestamp)
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_bad_colname
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_bad_colname
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_bad_colname
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
-# IMPALA-1104: Test computing stats on Avro tables created by Impala
-# with mismatched column definitions and Avro schema. Mismatched column type.
-create table avro_impala_alltypes_bad_coltype
-(id int, bool_col boolean, tinyint_col int, smallint_col int, int_col int,
-bigint_col bigint, float_col float, double_col bigint, date_string_col string,
-string_col string, timestamp_col timestamp)
-partitioned by (year int, month int)
-with serdeproperties
-('avro.schema.url'='$FILESYSTEM_PREFIX/test-warehouse/avro_schemas/functional/alltypes.json')
-stored as avro;
-====
----- QUERY
-compute stats avro_impala_alltypes_bad_coltype
----- RESULTS
-'Updated 0 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show table stats avro_impala_alltypes_bad_coltype
----- LABELS
-YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS, LOCATION
----- RESULTS
-'Total','',0,0,'0B','0B','','','',''
----- TYPES
-STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
-====
----- QUERY
-show column stats avro_impala_alltypes_bad_coltype
----- LABELS
-COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
-'year','INT',0,0,4,4
-'month','INT',0,0,4,4
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
# IMPALA-883: Compute table stats for an empty partition.
create table empty_partitioned (i int) partitioned by (j int);
alter table empty_partitioned add partition (j=1);
@@ -1154,33 +805,6 @@ COLUMN, TYPE, #DISTINCT VALUES, #NULLS, MAX SIZE, AVG SIZE
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
====
---- QUERY
-# Test that compute stats on a Hive-created Avro table without column defs
-# works (HIVE-6308, IMPALA-867).
-create table alltypes_no_coldef like functional_avro_snap.alltypes_no_coldef;
-compute stats alltypes_no_coldef
----- RESULTS
-'Updated 1 partition(s) and 11 column(s).'
----- TYPES
-STRING
-====
----- QUERY
-show column stats alltypes_no_coldef
----- RESULTS
-'id','INT',0,0,4,4
-'bool_col','BOOLEAN',2,0,1,1
-'tinyint_col','INT',0,0,4,4
-'smallint_col','INT',0,0,4,4
-'int_col','INT',0,0,4,4
-'bigint_col','BIGINT',0,0,8,8
-'float_col','FLOAT',0,0,4,4
-'double_col','DOUBLE',0,0,8,8
-'date_string_col','STRING',0,0,0,0
-'string_col','STRING',0,0,0,0
-'timestamp_col','STRING',0,0,0,0
----- TYPES
-STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
-====
----- QUERY
# Test that compute stats works on wide tables.
create table widetable_1000_cols
stored as parquet
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 027e19a..97159ab 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -21,7 +21,7 @@ describe database extended hive_test_desc_db
---- RESULTS
'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
'Parameter: ','',''
-'','test','USER'
+'','$USER','USER'
'Owner: ','',''
'','e','2.82'
'','pi','3.14'
@@ -72,4 +72,4 @@ describe database formatted impala_test_desc_db4
'impala_test_desc_db4','$NAMENODE/test2.db','test comment'
---- TYPES
string, string, string
-====
\ No newline at end of file
+====
diff --git a/tests/catalog_service/test_hms_failure.py b/tests/catalog_service/test_hms_failure.py
index d98f0b4..4e47b39 100644
--- a/tests/catalog_service/test_hms_failure.py
+++ b/tests/catalog_service/test_hms_failure.py
@@ -58,16 +58,15 @@ class TestHiveMetaStoreFailure(ImpalaTestSuite):
# Make sure the metastore is running even if the test aborts somewhere unexpected
# before restarting the metastore itself.
cls.run_hive_server()
- cls.client.execute("invalidate metadata")
super(TestHiveMetaStoreFailure, cls).teardown_class()
@pytest.mark.execute_serially
def test_hms_service_dies(self, vector):
"""Regression test for IMPALA-823 to verify the catalog service works properly when
HMS connections fail"""
- # Force all the tables to be reloaded and then kill the hive metastore.
+ # Force the tables to be uncached and then kill the hive metastore.
tbl_name = "functional.alltypes"
- self.client.execute("invalidate metadata")
+ self.client.execute("invalidate metadata %s" % tbl_name)
kill_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/kill-hive-server.sh')
check_call([kill_cmd], close_fds=True)
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 17b2329..f5df0a1 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -26,6 +26,9 @@ test_start_cluster_args = os.environ.get("TEST_START_CLUSTER_ARGS", "")
IMPALA_HOME = os.environ.get("IMPALA_HOME", "")
IMPALA_REMOTE_URL = os.environ.get("IMPALA_REMOTE_URL", "")
+# Default web UI URL for local test cluster
+DEFAULT_LOCAL_WEB_UI_URL = "http://localhost:25000"
+
# Find the likely BuildType of the running Impala. Assume it's found through the path
# $IMPALA_HOME/be/build/latest as a fallback.
build_type_arg_regex = re.compile(r'--build_type=(\w+)', re.I)
@@ -118,7 +121,7 @@ class ImpalaTestClusterFlagsDetector:
LOG.debug("Unable to read .cmake_build_type file, fetching build flags from " +
"web ui on localhost")
build_type, build_shared_libs = ImpalaTestClusterFlagsDetector.detect_using_web_ui(
- "http://localhost:25000")
+ DEFAULT_LOCAL_WEB_UI_URL)
library_link_type = LinkTypes.STATIC if build_shared_libs == "off"\
else LinkTypes.DYNAMIC
@@ -186,6 +189,7 @@ class ImpalaTestClusterProperties(object):
self._build_flavor = build_flavor
self._library_link_type = library_link_type
self._local_or_remote_build = local_or_remote_build
+ self._runtime_flags = None # Lazily populated to avoid unnecessary web UI calls.
@property
def build_flavor(self):
@@ -261,13 +265,37 @@ class ImpalaTestClusterProperties(object):
"""
return self._local_or_remote_build == REMOTE_BUILD
+ def _get_flags_from_web_ui(self, impala_url):
+ if self._runtime_flags is not None:
+ return self._runtime_flags
+ """Return the command line flags from the impala web UI. Returns a Python map with
+ the flag name as the key and a dictionary of flag properties as the value."""
+ response = requests.get(impala_url + "/varz?json")
+ assert response.status_code == requests.codes.ok,\
+ "Offending url: " + impala_url
+ assert "application/json" in response.headers['Content-Type']
+ self._runtime_flags = {}
+ for flag_dict in json.loads(response.text)["flags"]:
+ self._runtime_flags[flag_dict["name"]] = flag_dict
+ return self._runtime_flags
+
+ def is_catalog_v2_cluster(self):
+ """Whether we use CATALOG_V2 options, including local catalog and HMS notifications.
+ For now, assume that --use_local_catalog=true implies that the others are enabled."""
+ flags = self._get_flags_from_web_ui(web_ui_url)
+ key = "use_local_catalog"
+ # --use_local_catalog is hidden so does not appear in JSON if disabled.
+ return key in flags and flags[key]["current"] == "true"
+
if IMPALA_REMOTE_URL:
+ web_ui_url = IMPALA_REMOTE_URL
build_flavor, link_type =\
ImpalaTestClusterFlagsDetector.detect_using_web_ui(IMPALA_REMOTE_URL)
IMPALA_TEST_CLUSTER_PROPERTIES =\
ImpalaTestClusterProperties(build_flavor, link_type, REMOTE_BUILD)
else:
+ web_ui_url = DEFAULT_LOCAL_WEB_UI_URL
build_flavor, link_type =\
ImpalaTestClusterFlagsDetector.detect_using_build_root_or_web_ui(IMPALA_HOME)
IMPALA_TEST_CLUSTER_PROPERTIES =\
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 8b1f54f..7fe8216 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -883,6 +883,20 @@ class ImpalaTestSuite(BaseTestSuite):
actual_state))
return actual_state
+ def wait_for_db_to_appear(self, db_name, timeout_s):
+ """Wait until the database with 'db_name' is present in the impalad's local catalog.
+ Fail after timeout_s if the doesn't appear."""
+ start_time = time.time()
+ while time.time() - start_time < timeout_s:
+ try:
+ # This will throw an exception if the database is not present.
+ self.client.execute("describe database `{db_name}`".format(db_name=db_name))
+ return
+ except Exception:
+ time.sleep(0.2)
+ continue
+ raise Exception("Table {0} didn't show up after {1}s", db_name, timeout_s)
+
def assert_impalad_log_contains(self, level, line_regex, expected_count=1):
"""
Convenience wrapper around assert_log_contains for impalad logs.
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 95a62da..cad1caa 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -196,3 +196,19 @@ class SkipIfDockerizedCluster:
reason="IMPALA-4865: JVM hits OOM for large string. Heap is smaller in docker.")
insert_acls = pytest.mark.skipif(IS_DOCKERIZED_TEST_CLUSTER,
reason="IMPALA-8384: insert ACL tests are broken on dockerised minicluster.")
+
+
+class SkipIfCatalogV2:
+ """Expose decorators as methods so that is_catalog_v2_cluster() can be evaluated lazily
+ when needed, instead of whenever this module is imported."""
+ @classmethod
+ def stats_pulling_disabled(self):
+ return pytest.mark.skipif(
+ IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
+ reason="Local catalog does not use incremental stats pulling.")
+
+ @classmethod
+ def catalog_v1_test(self):
+ return pytest.mark.skipif(
+ IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
+ reason="Test is specific to old implementation of catalog.")
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index 5673438..7ae05f6 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -18,9 +18,11 @@
import pytest
from subprocess import check_call
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
+ SkipIfLocal, SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_single_exec_option_dimension,
@@ -47,9 +49,28 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfS3.eventually_consistent
def test_compute_stats(self, vector, unique_database):
self.run_test_case('QueryTest/compute-stats', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_avro(self, vector, unique_database):
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # IMPALA-7308: changed behaviour of various Avro edge cases significantly in the
+ # local catalog - the expected behaviour is different.
+ self.run_test_case('QueryTest/compute-stats-avro-catalog-v2', vector,
+ unique_database)
+ else:
+ self.run_test_case('QueryTest/compute-stats-avro', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_decimal(self, vector, unique_database):
# Test compute stats on decimal columns separately so we can vary between platforms
# with and without write support for decimals (Hive < 0.11 and >= 0.11).
self.run_test_case('QueryTest/compute-stats-decimal', vector, unique_database)
+
+ @SkipIfLocal.hdfs_blocks
+ @SkipIfS3.eventually_consistent
+ def test_compute_stats_date(self, vector, unique_database):
# Test compute stats on date columns separately.
self.run_test_case('QueryTest/compute-stats-date', vector, unique_database)
@@ -142,6 +163,7 @@ class TestComputeStats(ImpalaTestSuite):
assert("1\tpval\t8" in show_result.data[0])
@SkipIfS3.eventually_consistent
+ @SkipIfCatalogV2.stats_pulling_disabled()
def test_pull_stats_profile(self, vector, unique_database):
"""Checks that the frontend profile includes metrics when computing
incremental statistics.
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 800799c..71a0879 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -29,6 +29,7 @@ import random
import string
from subprocess import call
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (
@@ -62,7 +63,12 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
self.run_stmt_in_hive("drop database if exists hms_sanity_db cascade")
self.run_stmt_in_hive("create database hms_sanity_db")
# Make sure Impala's metadata is in sync.
- self.client.execute("invalidate metadata")
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # Using local catalog + HMS event processor - wait until the database shows up.
+ self.wait_for_db_to_appear("hms_sanity_db", timeout_s=30)
+ else:
+ # Using traditional catalog - need to invalidate to pick up hive-created db.
+ self.client.execute("invalidate metadata")
# Creating a database with the same name using 'IF NOT EXISTS' in Impala should
# not fail
self.client.execute("create database if not exists hms_sanity_db")
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 81fd60d..5633ba0 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -21,6 +21,7 @@ import pytest
import re
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfIsilon, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import ALL_NODES_ONLY
@@ -154,7 +155,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
- @pytest.mark.execute_serially # because of invalidate metadata
+ @pytest.mark.execute_serially # because of use of hardcoded database
def test_describe_db(self, vector):
self.__test_describe_db_cleanup()
try:
@@ -167,8 +168,12 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
"location \"" + get_fs_path("/test2.db") + "\"")
self.run_stmt_in_hive("create database hive_test_desc_db comment 'test comment' "
"with dbproperties('pi' = '3.14', 'e' = '2.82')")
- self.run_stmt_in_hive("alter database hive_test_desc_db set owner user test")
- self.client.execute("invalidate metadata")
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # Using local catalog + HMS event processor - wait until the database shows up.
+ self.wait_for_db_to_appear("hive_test_desc_db", timeout_s=30)
+ else:
+ # Using traditional catalog - need to invalidate to pick up hive-created db.
+ self.client.execute("invalidate metadata")
self.run_test_case('QueryTest/describe-db', vector)
finally:
self.__test_describe_db_cleanup()
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index 4b2c67b..6d9c058 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -14,6 +14,7 @@
from subprocess import check_call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -133,7 +134,11 @@ class TestRefreshPartition(ImpalaTestSuite):
% table_name)
# Make sure its still shows the same result before refreshing
result = self.client.execute("select count(*) from %s" % table_name)
- assert result.data == [str('0')]
+ valid_counts = [0]
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ # HMS notifications may pick up added partition racily.
+ valid_counts.append(1)
+ assert int(result.data[0]) in valid_counts
self.client.execute('refresh %s partition (y=333, z=5309)' % table_name)
assert '2\t333\t5309' == self.client.execute(
diff --git a/tests/query_test/test_avro_schema_resolution.py b/tests/query_test/test_avro_schema_resolution.py
index e660101..52feb8a 100644
--- a/tests/query_test/test_avro_schema_resolution.py
+++ b/tests/query_test/test_avro_schema_resolution.py
@@ -16,6 +16,7 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfCatalogV2
# This test requires that testdata/avro_schema_resolution/create_table.sql has been run
class TestAvroSchemaResolution(ImpalaTestSuite):
@@ -47,6 +48,7 @@ class TestAvroSchemaResolution(ImpalaTestSuite):
for x in range(len(result.data)):
assert comparison.data[x] == result.data[x]
+ @SkipIfCatalogV2.catalog_v1_test()
def test_avro_schema_changes(self, vector, unique_database):
"""Test for IMPALA-3314 and IMPALA-3513: Impalad shouldn't crash with stale Avro
metadata. Instead, should provide a meaningful error message.
@@ -54,5 +56,4 @@ class TestAvroSchemaResolution(ImpalaTestSuite):
... ADD COLUMN ...
Test for IMPALA-3776: Fix describe formatted when changing Avro schema.
"""
- # TODO(todd): skip the "stale metadata" tests if LocalCatalog is enabled
self.run_test_case('QueryTest/avro-schema-changes', vector, unique_database)
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index cb165d4..2960599 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -15,8 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
-from tests.common.environ import ImpalaTestClusterFlagsDetector
+from tests.common.environ import (IMPALA_TEST_CLUSTER_PROPERTIES,
+ ImpalaTestClusterFlagsDetector)
from tests.common.skip import SkipIfBuildType
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -53,6 +53,7 @@ class TestWebPage(ImpalaTestSuite):
# one with it.
TEST_PORTS_WITHOUT_SS = ["25000", "25020"]
TEST_PORTS_WITH_SS = ["25000", "25010", "25020"]
+ IMPALAD_TEST_PORT = ["25000"]
CATALOG_TEST_PORT = ["25020"]
def test_get_root_url(self):
@@ -287,14 +288,22 @@ class TestWebPage(ImpalaTestSuite):
def __test_catalog_object(self, db_name, tbl_name):
"""Tests the /catalog_object endpoint for the given db/table. Runs
against an unloaded as well as a loaded table."""
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster():
+ impalad_expected_str = \
+ "UnsupportedOperationException: LocalCatalog.getTCatalogObject"
+ else:
+ impalad_expected_str = tbl_name
+ obj_url = self.CATALOG_OBJECT_URL + \
+ "?object_type=TABLE&object_name={0}.{1}".format(db_name, tbl_name)
self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name))
- self.get_and_check_status(self.CATALOG_OBJECT_URL +
- "?object_type=TABLE&object_name=%s.%s" % (db_name, tbl_name), tbl_name,
- ports_to_test=self.TEST_PORTS_WITHOUT_SS)
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
self.client.execute("select count(*) from %s.%s" % (db_name, tbl_name))
- self.get_and_check_status(self.CATALOG_OBJECT_URL +
- "?object_type=TABLE&object_name=%s.%s" % (db_name, tbl_name), tbl_name,
- ports_to_test=self.TEST_PORTS_WITHOUT_SS)
+
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
def __test_table_metrics(self, db_name, tbl_name, metric):
self.client.execute("refresh %s.%s" % (db_name, tbl_name))