[impala] branch master updated: IMPALA-9351: Fix tests depending on hard-coded file paths of managed tables

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


The following commit(s) were added to refs/heads/master by this push:
     new 7b44b35  IMPALA-9351: Fix tests depending on hard-coded file paths of managed tables
7b44b35 is described below

commit 7b44b351321969fa6a90212f4c9b56c521a85ec4
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Fri Sep 11 14:20:19 2020 +0800

    IMPALA-9351: Fix tests depending on hard-coded file paths of managed tables
    
    Some tests (e.g. AnalyzeDDLTest.TestCreateTableLikeFileOrc) depend on
    hard-coded file paths of managed tables, assuming that there is always a
    file named 'base_0000001/bucket_00000_0' under the table dir. However,
    the file name is in the form of bucket_${bucket-id}_${attempt-id}. The
    last part of the file name is not guaranteed to be 0. If the first
    attempt fails and the second attempt succeeds, the file name will be
    bucket_00000_1.
    
    This patch replaces these hard-coded file paths to corresponding files
    that are uploaded to HDFS by commands. For tests that do need to use the
    file paths of managed table files, we do a listing on the table dir to
    get the file names, instead of hard-coding the file paths.
    
    Updated chars-formats.orc to contain column names in the file so can be
    used in more tests. The original one only has names like col0, col1,
    col2.
    
    Tests:
     - Run CORE tests
    
    Change-Id: Ie3136ee90e2444c4a12f0f2e1470fca1d5deaba0
    Reviewed-on: http://gerrit.cloudera.org:8080/16441
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../org/apache/impala/analysis/AnalyzeDDLTest.java    |   5 ++---
 testdata/bin/create-load-data.sh                      |   3 +++
 testdata/data/chars-formats.orc                       | Bin 1411 -> 1338 bytes
 .../queries/QueryTest/create-table-like-file-orc.test |  13 +++++--------
 tests/metadata/test_ddl.py                            |  11 ++++++++++-
 5 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 964162b..47ffe59 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2022,10 +2022,9 @@ public class AnalyzeDDLTest extends FrontendTestBase {
 
     // Inferring primitive and complex types
     AnalyzesOk("create table if not exists newtbl_DNE like orc " +
-        "'/test-warehouse/managed/alltypestiny_orc_def/year=2009/month=1/" +
-        "base_0000001/bucket_00000_0'");
+        "'/test-warehouse/schemas/alltypes_non_acid.orc'");
     AnalyzesOk("create table if not exists newtbl_DNE like orc " +
-        "'/test-warehouse/managed/complextypestbl_orc_def/base_0000001/bucket_00000_0'");
+        "'/test-warehouse/complextypestbl_non_transactional_orc_def/nullable.orc'");
 
     // check invalid paths
     AnalysisError("create table if not exists functional.zipcode_incomes like ORC " +
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index 5d2e0e3..312bd3a 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -213,6 +213,9 @@ function load-custom-schemas {
   ln -s ${IMPALA_HOME}/testdata/data/chars-formats.orc ${TMP_DIR}/chars_formats_orc_def
   ln -s ${IMPALA_HOME}/testdata/data/chars-formats.txt ${TMP_DIR}/chars_formats_text
 
+  # File used by CreateTableLikeOrc tests
+  ln -s ${IMPALA_HOME}/testdata/data/alltypes_non_acid.orc ${SCHEMA_TMP_DIR}
+
   hadoop fs -put -f ${TMP_DIR}/* /test-warehouse
 
   rm -r ${TMP_DIR}
diff --git a/testdata/data/chars-formats.orc b/testdata/data/chars-formats.orc
index 625c2c8..7703435 100644
Binary files a/testdata/data/chars-formats.orc and b/testdata/data/chars-formats.orc differ
diff --git a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
index 3d36299..85d3f1a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/create-table-like-file-orc.test
@@ -23,19 +23,16 @@ STRING, STRING, STRING
 ====
 ---- QUERY
 create table $DATABASE.temp_chars_table like ORC
-'$NAMENODE/$MANAGED_WAREHOUSE_DIR/chars_tiny_orc_def/base_0000001/bucket_00000_0'
+'$NAMENODE/test-warehouse/chars_formats_orc_def/chars-formats.orc'
 ---- RESULTS
 'Table has been created.'
 ====
 ---- QUERY
 describe $DATABASE.temp_chars_table
 ---- RESULTS
-'operation','int','Inferred from ORC file.'
-'originaltransaction','bigint','Inferred from ORC file.'
-'rowid','bigint','Inferred from ORC file.'
-'bucket','int','Inferred from ORC file.'
-'currenttransaction','bigint','Inferred from ORC file.'
-'row','struct<\n  cs:char(5),\n  cl:char(140),\n  vc:varchar(32)\n>','Inferred from ORC file.'
+'cs','char(5)','Inferred from ORC file.'
+'cl','char(140)','Inferred from ORC file.'
+'vc','varchar(32)','Inferred from ORC file.'
 ---- TYPES
 STRING, STRING, STRING
 ====
@@ -115,7 +112,7 @@ STRING, STRING, STRING
 ====
 ---- QUERY
 create external table transactional_complextypes_clone like ORC
-'$NAMENODE/$MANAGED_WAREHOUSE_DIR/complextypestbl_orc_def/base_0000001/bucket_00000_0'
+'$TRANSACTIONAL_COMPLEXTYPESTBL_FILE'
 stored as orc;
 ---- RESULTS
 'Table has been created.'
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index f6282cb..455a1da 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -36,6 +36,7 @@ from tests.util.filesystem_utils import (
     IS_ADLS,
     FILESYSTEM_NAME)
 from tests.common.impala_cluster import ImpalaCluster
+from tests.util.filesystem_utils import FILESYSTEM_PREFIX
 
 # Validates DDL statements (create, drop)
 class TestDdlStatements(TestDdlBase):
@@ -298,9 +299,17 @@ class TestDdlStatements(TestDdlBase):
   @SkipIfS3.hive
   @UniqueDatabase.parametrize(sync_ddl=True)
   def test_create_table_like_file_orc(self, vector, unique_database):
+    COMPLEXTYPETBL_PATH = 'test-warehouse/managed/complextypestbl_orc_def/'
+    base_dir = filter(lambda s: s.startswith('base'),
+      self.filesystem_client.ls(COMPLEXTYPETBL_PATH))[0]
+    bucket_file = filter(lambda s: s.startswith('bucket'),
+      self.filesystem_client.ls(COMPLEXTYPETBL_PATH + base_dir))[0]
     vector.get_value('exec_option')['abort_on_error'] = False
     self.run_test_case('QueryTest/create-table-like-file-orc', vector,
-        use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
+        use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector),
+        test_file_vars={
+          '$TRANSACTIONAL_COMPLEXTYPESTBL_FILE':
+          FILESYSTEM_PREFIX + '/' + COMPLEXTYPETBL_PATH + base_dir + '/' + bucket_file})
 
   @UniqueDatabase.parametrize(sync_ddl=True)
   def test_create_table_as_select(self, vector, unique_database):