[impala] 01/02: IMPALA-9117, IMPALA-7726: Fixed a few unit tests for ABFS

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stakiar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit e8fda1f224d3ad237183a53e238eee90188d82e2
Author: Sahil Takiar <ta...@gmail.com>
AuthorDate: Fri Nov 1 21:29:00 2019 +0000

    IMPALA-9117, IMPALA-7726: Fixed a few unit tests for ABFS
    
    This test makes the following changes / fixes when running Impala tests
    on ABFS:
    * Skips some tests in test_lineage.py that don't work on ABFS / ADLS
    (they were already skipped for S3)
    * Skips some tests in test_mt_dop.py; the test creates a directory that
    ends with a period (and ABFS does not support writing files or
    directories that end with a period)
    * Removes the ABFS skip flag SkipIfABFS.trash (IMPALA-7726: Drop with
    purge tests fail against ABFS due to trash misbehavior"); I removed
    these flags and looped the tests overnight with no failures, so it is
    likely whatever bug was causing this has now been fixed
    * Now that HADOOP-15860 has been resolved, and the agreed upon behavior
    for ABFS is that it will fail if a client tries to write a file /
    directory that ends with a period, I added a new entry to the SkipIfABFS
    class called file_or_folder_name_ends_with_period and applied it where
    necessary
    
    Testing:
    * Ran core tests on ABFS
    
    Change-Id: I18ae5b0f7de6aa7628a1efd780ff30a0cc3c5285
    Reviewed-on: http://gerrit.cloudera.org:8080/14636
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 tests/common/skip.py                 | 2 ++
 tests/custom_cluster/test_lineage.py | 4 +++-
 tests/custom_cluster/test_mt_dop.py  | 3 ++-
 tests/metadata/test_ddl.py           | 2 --
 tests/query_test/test_insert.py      | 2 +-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/common/skip.py b/tests/common/skip.py
index 1426b86..5203de5 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -75,6 +75,8 @@ class SkipIfABFS:
       reason="HDFS encryption is not supported with ABFS")
   trash = pytest.mark.skipif(IS_ABFS,
       reason="Drop/purge not working as expected on ABFS, IMPALA-7726")
+  file_or_folder_name_ends_with_period = pytest.mark.skipif(IS_ABFS,
+      reason="ABFS does not support file / directories that end with a period")
 
   # These need test infra work to re-enable.
   udfs = pytest.mark.skipif(IS_ABFS, reason="udas/udfs not copied to ABFS")
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 850ae65..73f785d 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -26,7 +26,7 @@ import tempfile
 import time
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3
+from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3
 
 LOG = logging.getLogger(__name__)
 
@@ -132,6 +132,8 @@ class TestLineage(CustomClusterTestSuite):
           assert lineage_json["queryText"] == query
           assert lineage_json["tableLocation"] is not None
 
+  @SkipIfABFS.hbase
+  @SkipIfADLS.hbase
   @SkipIfS3.hbase
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
diff --git a/tests/custom_cluster/test_mt_dop.py b/tests/custom_cluster/test_mt_dop.py
index 0bf8a1b..d4f664e 100644
--- a/tests/custom_cluster/test_mt_dop.py
+++ b/tests/custom_cluster/test_mt_dop.py
@@ -20,7 +20,7 @@ from copy import deepcopy
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 from tests.common.environ import build_flavor_timeout
-from tests.common.skip import SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfABFS, SkipIfNotHdfsMinicluster
 
 WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=100000)
 
@@ -36,6 +36,7 @@ class TestMtDopFlags(CustomClusterTestSuite):
 
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(impalad_args="--unlock_mt_dop=true")
+  @SkipIfABFS.file_or_folder_name_ends_with_period
   def test_mt_dop_all(self, vector, unique_database):
     """Test joins, inserts and runtime filters with mt_dop > 0"""
     vector = deepcopy(vector)
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 3a3c982..c51e0f4 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -40,7 +40,6 @@ from tests.common.impala_cluster import ImpalaCluster
 # Validates DDL statements (create, drop)
 class TestDdlStatements(TestDdlBase):
   @SkipIfLocal.hdfs_client
-  @SkipIfABFS.trash
   def test_drop_table_with_purge(self, unique_database):
     """This test checks if the table data is permanently deleted in
     DROP TABLE <tbl> PURGE queries"""
@@ -451,7 +450,6 @@ class TestDdlStatements(TestDdlBase):
         use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
 
   @SkipIfLocal.hdfs_client
-  @SkipIfABFS.trash
   def test_drop_partition_with_purge(self, vector, unique_database):
     """Verfies whether alter <tbl> drop partition purge actually skips trash"""
     self.client.execute(
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 173a272..7807030 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -129,7 +129,7 @@ class TestInsertQueries(ImpalaTestSuite):
   # Erasure coding doesn't respect memory limit
   @SkipIfEC.fix_later
   # ABFS partition names cannot end in periods
-  @SkipIfABFS.jira(reason="HADOOP-15860")
+  @SkipIfABFS.file_or_folder_name_ends_with_period
   def test_insert(self, vector):
     if (vector.get_value('table_format').file_format == 'parquet'):
       vector.get_value('exec_option')['COMPRESSION_CODEC'] = \