You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/04/16 22:41:13 UTC
[impala] branch master updated: IMPALA-8415: Fix tests broken by
storage layer information
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new c0a6aad IMPALA-8415: Fix tests broken by storage layer information
c0a6aad is described below
commit c0a6aad28dd1efaf388f0cfeaa056995213d56e5
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Mon Apr 15 15:35:49 2019 -0700
IMPALA-8415: Fix tests broken by storage layer information
Storage layer information was added to the query profile by
IMPALA-6050. This broke some tests on exhaustive and s3 runs
due to changes in formatting.
This fixes the issues:
1. Replace HDFS SCAN with $FILESYSTEM_NAME SCAN in some test files
2. Add $FILESYSTEM_NAME to partition information string
Testing:
- Ran exhaustive HDFS tests
- Ran s3 tests
Change-Id: I11c6ab9c888464a0f0daaf8a7a6f565d25731872
Reviewed-on: http://gerrit.cloudera.org:8080/13025
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../queries/QueryTest/corrupt-stats.test | 28 +++++++++++-----------
.../functional-query/queries/QueryTest/set.test | 12 +++++-----
.../queries/QueryTest/stats-extrapolation.test | 10 ++++----
tests/metadata/test_ddl.py | 16 +++++++++----
4 files changed, 36 insertions(+), 30 deletions(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test b/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
index 65409b5..00af63e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/corrupt-stats.test
@@ -65,8 +65,8 @@ explain select count(*) from corrupted where org = 1;
'01:AGGREGATE'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted]'
-' partitions=1/2 files=1 size=24B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted]'
+' $FILESYSTEM_NAME partitions=1/2 files=1 size=24B'
' row-size=0B cardinality=0'
---- TYPES
STRING
@@ -92,8 +92,8 @@ explain select count(*) from corrupted;
'01:AGGREGATE [FINALIZE]'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted]'
-' partitions=2/2 files=2 size=48B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted]'
+' $FILESYSTEM_NAME partitions=2/2 files=2 size=48B'
' row-size=0B cardinality=6'
---- TYPES
STRING
@@ -118,8 +118,8 @@ explain select count(*) from corrupted;
'01:AGGREGATE [FINALIZE]'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted]'
-' partitions=2/2 files=2 size=48B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted]'
+' $FILESYSTEM_NAME partitions=2/2 files=2 size=48B'
' row-size=0B cardinality=6'
---- TYPES
STRING
@@ -141,8 +141,8 @@ explain select count(*) from corrupted where org = 2;
'01:AGGREGATE'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted]'
-' partitions=1/2 files=1 size=24B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted]'
+' $FILESYSTEM_NAME partitions=1/2 files=1 size=24B'
' row-size=0B cardinality=6'
---- TYPES
STRING
@@ -168,8 +168,8 @@ explain select count(*) from corrupted_no_part;
'01:AGGREGATE [FINALIZE]'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted_no_part]'
-' partitions=1/1 files=1 size=6B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted_no_part]'
+' $FILESYSTEM_NAME partitions=1/1 files=1 size=6B'
' row-size=0B cardinality=3'
---- TYPES
STRING
@@ -201,8 +201,8 @@ explain select count(*) from corrupted_no_part;
'01:AGGREGATE'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted_no_part]'
-' partitions=1/1 files=1 size=6B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted_no_part]'
+' $FILESYSTEM_NAME partitions=1/1 files=1 size=6B'
' row-size=0B cardinality=0'
---- TYPES
STRING
@@ -224,8 +224,8 @@ explain select count(*) from corrupted_no_part;
'01:AGGREGATE'
'| output: count(*)'
'|'
-'00:SCAN HDFS [$DATABASE.corrupted_no_part]'
-' partitions=1/1 files=1 size=6B'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.corrupted_no_part]'
+' $FILESYSTEM_NAME partitions=1/1 files=1 size=6B'
' row-size=0B cardinality=unavailable'
---- TYPES
STRING
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 32a93f5..3acf9ac 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -191,7 +191,7 @@ explain select min(month), max(year), ndv(day) from functional.alltypesagg;
'03:AGGREGATE [FINALIZE]'
'02:EXCHANGE [UNPARTITIONED]'
'01:AGGREGATE'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set explain_level=0;
@@ -201,7 +201,7 @@ explain select min(month), max(year), ndv(day) from functional.alltypesagg;
'03:AGGREGATE [FINALIZE]'
'02:EXCHANGE [UNPARTITIONED]'
'01:AGGREGATE'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set explain_level=0;
@@ -214,7 +214,7 @@ explain select count(distinct double_col) from functional.alltypesagg;
'04:AGGREGATE'
'03:EXCHANGE [HASH(double_col)]'
'01:AGGREGATE [STREAMING]'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set explain_level=0;
@@ -227,7 +227,7 @@ explain select count(distinct double_col) from functional.alltypesagg;
'04:AGGREGATE'
'03:EXCHANGE [HASH(double_col)]'
'01:AGGREGATE [STREAMING]'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set explain_level=0;
@@ -240,7 +240,7 @@ explain select count(distinct double_col) from functional.alltypesagg;
'04:AGGREGATE'
'03:EXCHANGE [HASH(double_col)]'
'01:AGGREGATE'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set explain_level=0;
@@ -253,7 +253,7 @@ explain select count(distinct double_col) from functional.alltypesagg;
'04:AGGREGATE'
'03:EXCHANGE [HASH(double_col)]'
'01:AGGREGATE'
-'00:SCAN HDFS [functional.alltypesagg]'
+'00:SCAN $FILESYSTEM_NAME [functional.alltypesagg]'
====
---- QUERY
set max_row_size=-1;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
index 3c1e31f..c784899 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
@@ -62,7 +62,7 @@ row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
-'00:SCAN HDFS [$DATABASE.alltypes]'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=12 size=.*
' stored statistics:'
row_regex:.*table: rows=3.65K size=.*
@@ -90,7 +90,7 @@ row_regex:.*mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
-'00:SCAN HDFS [$DATABASE.alltypes]'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.alltypes]'
' partition predicates: `month` IN (CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT))'
row_regex:.*partitions=3/12 files=3 size=.*
' stored statistics:'
@@ -119,7 +119,7 @@ row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
-'00:SCAN HDFS [$DATABASE.alltypes]'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=24 size=.*
' stored statistics:'
row_regex:.*table: rows=3.65K size=.*
@@ -149,7 +149,7 @@ row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
-'00:SCAN HDFS [$DATABASE.alltypes]'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.alltypes]'
' partition predicates: `year` = CAST(2010 AS INT)'
row_regex:.*partitions=12/24 files=12 size=.*
' stored statistics:'
@@ -179,7 +179,7 @@ row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
'PLAN-ROOT SINK'
row_regex:.*mem-estimate=.* mem-reservation=.*
'|'
-'00:SCAN HDFS [$DATABASE.alltypes]'
+'00:SCAN $FILESYSTEM_NAME [$DATABASE.alltypes]'
' partition predicates: `year` = CAST(2010 AS INT)'
row_regex:.*partitions=12/24 files=12 size=.*
' stored statistics:'
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index b696d95..c8871c5 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -27,7 +27,12 @@ from tests.common.impala_test_suite import LOG
from tests.common.parametrize import UniqueDatabase
from tests.common.skip import SkipIf, SkipIfABFS, SkipIfADLS, SkipIfKudu, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
-from tests.util.filesystem_utils import WAREHOUSE, IS_HDFS, IS_S3, IS_ADLS
+from tests.util.filesystem_utils import (
+ WAREHOUSE,
+ IS_HDFS,
+ IS_S3,
+ IS_ADLS,
+ FILESYSTEM_NAME)
from tests.common.impala_cluster import ImpalaCluster
# Validates DDL statements (create, drop)
@@ -509,16 +514,17 @@ class TestDdlStatements(TestDdlBase):
# Test the plan to make sure hints were applied correctly
plan = self.execute_query("explain select * from %s.hints_test" % unique_database,
query_options={'explain_level':0})
- assert """PLAN-ROOT SINK
+ plan_match = """PLAN-ROOT SINK
08:EXCHANGE [UNPARTITIONED]
04:HASH JOIN [INNER JOIN, PARTITIONED]
|--07:EXCHANGE [HASH(c.id)]
-| 02:SCAN HDFS [functional.alltypessmall c]
+| 02:SCAN {filesystem_name} [functional.alltypessmall c]
06:EXCHANGE [HASH(b.id)]
03:HASH JOIN [INNER JOIN, BROADCAST]
|--05:EXCHANGE [BROADCAST]
-| 01:SCAN HDFS [functional.alltypes b]
-00:SCAN HDFS [functional.alltypestiny a]""" in '\n'.join(plan.data)
+| 01:SCAN {filesystem_name} [functional.alltypes b]
+00:SCAN {filesystem_name} [functional.alltypestiny a]"""
+ assert plan_match.format(filesystem_name=FILESYSTEM_NAME) in '\n'.join(plan.data)
def _verify_describe_view(self, vector, view_name, expected_substr):
"""