[impala] 03/04: IMPALA-8854: fix acid insert tests

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 3e9cac0cac0063dda5d73ce00ccc8bd2332f50a1
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Wed Aug 14 18:26:13 2019 +0200

    IMPALA-8854: fix acid insert tests
    
    test_acid_nonacid_insert has been failing lately. HMS became more
    strict about checking the capabilities of its clients. Seems like
    the Python client doesn't set any capabilities for itself therefore
    HMS rejects its attempts of creating and dropping tables.
    
    Now instead of using the RESET utility from the e2e test framework
    (to drop and re-create tables), the test is using a unique database
    and creates the tables through Impala. Different file formats are
    exercised with the help of the DEFAULT_FILE_FORMAT query option.
    
    Change-Id: I3a82338a7820d0ee748c961c8656fa3319c3929c
    Reviewed-on: http://gerrit.cloudera.org:8080/14064
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../functional/functional_schema_template.sql      | 23 --------
 .../queries/QueryTest/acid-insert.test             | 63 +++++++++++-----------
 tests/query_test/test_insert.py                    | 29 ++++------
 3 files changed, 41 insertions(+), 74 deletions(-)

diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index ab59a5c..d460eaa 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2649,26 +2649,3 @@ date_part DATE
 id_col INT
 date_col DATE
 ====
----- DATASET
-functional
----- BASE_TABLE_NAME
-insertonly_nopart_insert
----- HIVE_MAJOR_VERSION
-3
----- CREATE
-CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (i int)
-STORED AS {file_format}
-TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only');
-====
----- DATASET
-functional
----- BASE_TABLE_NAME
-insertonly_part_insert
----- HIVE_MAJOR_VERSION
-3
----- CREATE
-CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (i int)
-PARTITIONED BY (p int)
-STORED AS {file_format}
-TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only');
-====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
index 3e0baae..094b463 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/acid-insert.test
@@ -1,11 +1,11 @@
 ====
----- SETUP
-RESET insertonly_nopart_insert
 ---- QUERY
-insert into insertonly_nopart_insert values (1), (2);
+create table insertonly_nopart (i int)
+tblproperties('transactional'='true', 'transactional_properties'='insert_only');
+insert into insertonly_nopart values (1), (2);
 ====
 ---- QUERY
-select i from insertonly_nopart_insert order by i;
+select i from insertonly_nopart order by i;
 ---- RESULTS
 1
 2
@@ -13,10 +13,10 @@ select i from insertonly_nopart_insert order by i;
 INT
 ====
 ---- QUERY
-insert into insertonly_nopart_insert values (3);
+insert into insertonly_nopart values (3);
 ====
 ---- QUERY
-select i from insertonly_nopart_insert order by i;
+select i from insertonly_nopart order by i;
 ---- RESULTS
 1
 2
@@ -25,43 +25,44 @@ select i from insertonly_nopart_insert order by i;
 INT
 ====
 ---- QUERY
-insert overwrite insertonly_nopart_insert values (10);
+insert overwrite insertonly_nopart values (10);
 ====
 ---- QUERY
-select i from insertonly_nopart_insert order by i;
+select i from insertonly_nopart order by i;
 ---- RESULTS
 10
 ---- TYPES
 INT
 ====
 ---- QUERY
-insert overwrite insertonly_nopart_insert select 100;
+insert overwrite insertonly_nopart select 100;
 ====
 ---- QUERY
-select i from insertonly_nopart_insert order by i;
+select i from insertonly_nopart order by i;
 ---- RESULTS
 100
 ---- TYPES
 INT
 ====
 ---- QUERY
-insert overwrite insertonly_nopart_insert
-select * from insertonly_nopart_insert limit 0;
+insert overwrite insertonly_nopart
+select * from insertonly_nopart limit 0;
 ====
 ---- QUERY
-select i from insertonly_nopart_insert order by i;
+select i from insertonly_nopart order by i;
 ---- RESULTS
 ---- TYPES
 INT
 ====
----- SETUP
-RESET insertonly_part_insert
 ---- QUERY
-insert into insertonly_part_insert partition (p=1) values (10), (11);
-insert into insertonly_part_insert partition (p=2) values (20);
+create table if not exists insertonly_part (i int)
+partitioned by (p int)
+tblproperties('transactional'='true', 'transactional_properties'='insert_only');
+insert into insertonly_part partition (p=1) values (10), (11);
+insert into insertonly_part partition (p=2) values (20);
 ====
 ---- QUERY
-select p, i from insertonly_part_insert order by i;
+select p, i from insertonly_part order by i;
 ---- RESULTS
 1,10
 1,11
@@ -70,11 +71,11 @@ select p, i from insertonly_part_insert order by i;
 INT,INT
 ====
 ---- QUERY
-insert into insertonly_part_insert partition (p=2) values (21);
-insert into insertonly_part_insert partition (p=3) values (30);
+insert into insertonly_part partition (p=2) values (21);
+insert into insertonly_part partition (p=3) values (30);
 ====
 ---- QUERY
-select p, i from insertonly_part_insert order by i;
+select p, i from insertonly_part order by i;
 ---- RESULTS
 1,10
 1,11
@@ -85,11 +86,11 @@ select p, i from insertonly_part_insert order by i;
 INT,INT
 ====
 ---- QUERY
-insert overwrite insertonly_part_insert partition (p=2) values (22);
-insert overwrite insertonly_part_insert partition (p=3) values (31);
+insert overwrite insertonly_part partition (p=2) values (22);
+insert overwrite insertonly_part partition (p=3) values (31);
 ====
 ---- QUERY
-select p, i from insertonly_part_insert order by i;
+select p, i from insertonly_part order by i;
 ---- RESULTS
 1,10
 1,11
@@ -99,24 +100,24 @@ select p, i from insertonly_part_insert order by i;
 INT,INT
 ====
 ---- QUERY
-insert overwrite insertonly_part_insert partition (p=1)
-select * from insertonly_nopart_insert limit 0;
-insert overwrite insertonly_part_insert partition (p=2)
-select * from insertonly_nopart_insert limit 0;
+insert overwrite insertonly_part partition (p=1)
+select * from insertonly_nopart limit 0;
+insert overwrite insertonly_part partition (p=2)
+select * from insertonly_nopart limit 0;
 ====
 ---- QUERY
-select p, i from insertonly_part_insert order by i;
+select p, i from insertonly_part order by i;
 ---- RESULTS
 3,31
 ---- TYPES
 INT,INT
 ====
 ---- QUERY
-insert overwrite insertonly_part_insert partition (p)
+insert overwrite insertonly_part partition (p)
 values (1000, 1), (2000, 2), (4000, 4), (5000, 5), (5001, 5);
 ====
 ---- QUERY
-select p, i from insertonly_part_insert order by p, i;
+select p, i from insertonly_part order by p, i;
 ---- RESULTS
 1,1000
 2,2000
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index a630b3e..17ee7d8 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -136,27 +136,16 @@ class TestInsertQueries(ImpalaTestSuite):
     self.run_test_case('QueryTest/insert', vector,
         multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1)
 
-  @pytest.mark.execute_serially
   @SkipIfHive2.acid
-  def test_acid_insert(self, vector):
-    pytest.skip("IMPALA-8854: skipping to unbreak builds")
-    if (vector.get_value('table_format').file_format == 'parquet'):
-      vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
-          vector.get_value('compression_codec')
-    # We need to turn off capability checks. Otherwise we get an error from HMS because
-    # this python client doesn't have the capability for handling ACID tables. But we only
-    # need to drop and create such tables, and table properties are preserved during
-    # those operations and this is enough for the tests (A table is ACID if it has the
-    # relevant table properties).
-    CAPABILITY_CHECK_CONF = "hive.metastore.client.capability.check"
-    capability_check = self.hive_client.getMetaConf(CAPABILITY_CHECK_CONF)
-    try:
-      self.hive_client.setMetaConf(CAPABILITY_CHECK_CONF, "false")
-      self.run_test_case('QueryTest/acid-insert', vector,
-          multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1)
-    finally:
-      # Reset original state.
-      self.hive_client.setMetaConf(CAPABILITY_CHECK_CONF, capability_check)
+  @UniqueDatabase.parametrize(sync_ddl=True)
+  def test_acid_insert(self, vector, unique_database):
+    exec_options = vector.get_value('exec_option')
+    file_format = vector.get_value('table_format').file_format
+    if (file_format == 'parquet'):
+      exec_options['COMPRESSION_CODEC'] = vector.get_value('compression_codec')
+    exec_options['DEFAULT_FILE_FORMAT'] = file_format
+    self.run_test_case('QueryTest/acid-insert', vector, unique_database,
+        multiple_impalad=exec_options['sync_ddl'] == 1)
 
   @SkipIfHive2.acid
   @UniqueDatabase.parametrize(sync_ddl=True)