You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/08/11 05:48:30 UTC
[impala] 02/03: IMPALA-11456: Collapse filesystem Skip logic
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1eb0510eaa8b54e021847775db32e250775101d9
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Mon Jul 25 10:53:25 2022 -0700
IMPALA-11456: Collapse filesystem Skip logic
Combines all SkipIf* classes for different filesystems into a single
SkipIfFS class. Many cases are simplified to 'not IS_HDFS', with the
rest as filesystem-specific special cases. The 'jira' option is removed
in favor of specific flags for each issue.
Change-Id: Ib928a6274baaaec45614887b9e762346a25812a1
Reviewed-on: http://gerrit.cloudera.org:8080/18781
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/authorization/test_ranger.py | 13 +-
tests/common/skip.py | 164 +++------------------
tests/custom_cluster/test_admission_controller.py | 16 +-
tests/custom_cluster/test_coordinators.py | 12 +-
tests/custom_cluster/test_events_custom_configs.py | 11 +-
.../test_hive_parquet_codec_interop.py | 7 +-
.../custom_cluster/test_hive_text_codec_interop.py | 7 +-
tests/custom_cluster/test_insert_behaviour.py | 14 +-
tests/custom_cluster/test_lineage.py | 10 +-
tests/custom_cluster/test_local_catalog.py | 12 +-
tests/custom_cluster/test_local_tz_conversion.py | 11 +-
.../test_metadata_no_events_processing.py | 11 +-
tests/custom_cluster/test_metadata_replicas.py | 22 +--
.../custom_cluster/test_parquet_max_page_header.py | 11 +-
tests/custom_cluster/test_partition.py | 8 +-
tests/custom_cluster/test_permanent_udfs.py | 41 +-----
tests/custom_cluster/test_query_retries.py | 10 +-
tests/custom_cluster/test_restart_services.py | 8 +-
tests/custom_cluster/test_rpc_timeout.py | 5 +-
tests/custom_cluster/test_runtime_profile.py | 4 +-
.../custom_cluster/test_topic_update_frequency.py | 6 +-
tests/data_errors/test_data_errors.py | 28 +---
tests/failure/test_failpoints.py | 17 +--
tests/metadata/test_catalogd_debug_actions.py | 6 +-
tests/metadata/test_compute_stats.py | 31 +---
tests/metadata/test_ddl.py | 14 +-
tests/metadata/test_event_processing.py | 12 +-
tests/metadata/test_hdfs_encryption.py | 14 +-
tests/metadata/test_hdfs_permissions.py | 10 +-
tests/metadata/test_hms_integration.py | 24 +--
tests/metadata/test_metadata_query_statements.py | 23 +--
tests/metadata/test_partition_metadata.py | 14 +-
tests/metadata/test_recover_partitions.py | 4 +-
tests/metadata/test_recursive_listing.py | 15 +-
tests/metadata/test_refresh_partition.py | 12 +-
tests/metadata/test_stale_metadata.py | 5 +-
tests/metadata/test_stats_extrapolation.py | 6 +-
tests/metadata/test_views_compatibility.py | 15 +-
tests/query_test/test_acid.py | 130 ++--------------
tests/query_test/test_date_queries.py | 11 +-
tests/query_test/test_hbase_queries.py | 20 +--
tests/query_test/test_hdfs_caching.py | 42 +-----
tests/query_test/test_insert.py | 6 +-
tests/query_test/test_insert_behaviour.py | 67 ++-------
tests/query_test/test_insert_parquet.py | 26 +---
tests/query_test/test_join_queries.py | 20 +--
tests/query_test/test_mt_dop.py | 4 +-
tests/query_test/test_nested_types.py | 45 +-----
tests/query_test/test_observability.py | 22 +--
tests/query_test/test_partitioning.py | 13 +-
tests/query_test/test_resource_limits.py | 10 +-
tests/query_test/test_runtime_filters.py | 9 +-
tests/query_test/test_scanners.py | 97 ++----------
tests/stress/test_acid_stress.py | 14 +-
tests/stress/test_ddl_stress.py | 12 +-
55 files changed, 213 insertions(+), 988 deletions(-)
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index f68645576..049d3f4f1 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -27,9 +27,7 @@ from subprocess import check_call
from getpass import getuser
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfLocal, SkipIfHive2, SkipIfGCS, SkipIfCOS,
- SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfHive2
from tests.common.test_dimensions import (create_client_protocol_dimension,
create_exec_option_dimension, create_orc_dimension)
from tests.util.hdfs_util import NAMENODE
@@ -1402,14 +1400,7 @@ class TestRanger(CustomClusterTestSuite):
TestRanger._remove_policy(unique_name + str(i))
@pytest.mark.execute_serially
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@SkipIfHive2.ranger_auth
@CustomClusterTestSuite.with_args()
def test_hive_with_ranger_setup(self, vector):
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 0657155a0..70bb0717b 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -20,7 +20,6 @@
# annotate the class or test routine with the marker.
#
-import os
import pytest
from functools import partial
@@ -43,130 +42,39 @@ from tests.util.filesystem_utils import (
IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance()
-class SkipIfS3:
+class SkipIfFS:
# These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_S3, reason="SET CACHED not implemented for S3")
- hive = pytest.mark.skipif(IS_S3, reason="Hive doesn't work with S3")
- hdfs_block_size = pytest.mark.skipif(IS_S3, reason="S3 uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_S3, reason="HDFS acls are not supported on S3")
- jira = partial(pytest.mark.skipif, IS_S3)
- hdfs_encryption = pytest.mark.skipif(IS_S3,
- reason="HDFS encryption is not supported with S3")
+ hdfs_caching = pytest.mark.skipif(not IS_HDFS, reason="SET CACHED not implemented")
+ hdfs_encryption = pytest.mark.skipif(not IS_HDFS,
+ reason="HDFS encryption is not supported")
+ hdfs_block_size = pytest.mark.skipif(not IS_HDFS, reason="uses it's own block size")
+ hdfs_acls = pytest.mark.skipif(not IS_HDFS, reason="HDFS acls are not supported")
+
+ # Special case product limitations.
empty_directory = pytest.mark.skipif(IS_S3,
reason="Empty directories are not supported on S3")
-
- # These need test infra work to re-enable.
- udfs = pytest.mark.skipif(IS_S3, reason="udas/udfs not copied to S3")
- datasrc = pytest.mark.skipif(IS_S3, reason="data sources not copied to S3")
- hbase = pytest.mark.skipif(IS_S3, reason="HBase not started with S3")
- qualified_path = pytest.mark.skipif(IS_S3,
- reason="Tests rely on HDFS qualified paths, IMPALA-1872")
- iceberg = pytest.mark.skipif(IS_S3,
- reason="Currently Iceberg is only supported on HDFS.")
- variable_listing_times = pytest.mark.skipif(IS_S3,
- reason="Flakiness due to unpredictable listing times on S3.")
-
-
-class SkipIfOzone:
- # These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_OZONE, reason="SET CACHED not implemented for Ozone")
- hdfs_block_size = pytest.mark.skipif(IS_OZONE, reason="Ozone uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_OZONE, reason="HDFS acls are not supported on Ozone")
- hdfs_encryption = pytest.mark.skipif(IS_OZONE,
- reason="HDFS encryption is not supported with Ozone")
- no_storage_ids = pytest.mark.skipif(IS_OZONE,
- reason="Ozone does not return storage ids, IMPALA-10213")
-
- # These need test infra work to re-enable.
- hive = pytest.mark.skipif(IS_OZONE, reason="Hive not started with Ozone")
- hbase = pytest.mark.skipif(IS_OZONE, reason="HBase not started with Ozone")
- qualified_path = pytest.mark.skipif(IS_OZONE,
- reason="Tests rely on HDFS qualified paths, IMPALA-1872")
-
-
-class SkipIfABFS:
-
- # These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_ABFS, reason="SET CACHED not implemented for ABFS")
- hive = pytest.mark.skipif(IS_ABFS, reason="Hive doesn't work with ABFS")
- hdfs_block_size = pytest.mark.skipif(IS_ABFS, reason="ABFS uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_ABFS, reason="HDFS acls are not supported on ABFS")
- jira = partial(pytest.mark.skipif, IS_ABFS)
- hdfs_encryption = pytest.mark.skipif(IS_ABFS,
- reason="HDFS encryption is not supported with ABFS")
- trash = pytest.mark.skipif(IS_ABFS,
- reason="Drop/purge not working as expected on ABFS, IMPALA-7726")
file_or_folder_name_ends_with_period = pytest.mark.skipif(IS_ABFS,
reason="ABFS does not support file / directories that end with a period")
+ stress_insert_timeouts = pytest.mark.skipif(IS_COS or IS_GCS,
+ reason="IMPALA-10563, IMPALA-10773")
+ shutdown_idle_fails = pytest.mark.skipif(IS_COS or IS_GCS,
+ reason="IMPALA-10562")
+ late_filters = pytest.mark.skipif(IS_ISILON, reason="IMPALA-6998")
+ read_past_eof = pytest.mark.skipif(IS_S3 or IS_GCS, reason="IMPALA-2512")
+ no_storage_ids = pytest.mark.skipif(IS_OZONE,
+ reason="Ozone does not return storage ids, IMPALA-10213")
+ large_block_size = pytest.mark.skipif(IS_OZONE,
+ reason="block size is larger than 128MB")
# These need test infra work to re-enable.
- udfs = pytest.mark.skipif(IS_ABFS, reason="udas/udfs not copied to ABFS")
- datasrc = pytest.mark.skipif(IS_ABFS, reason="data sources not copied to ABFS")
- hbase = pytest.mark.skipif(IS_ABFS, reason="HBase not started with ABFS")
- qualified_path = pytest.mark.skipif(IS_ABFS,
- reason="Tests rely on HDFS qualified paths, IMPALA-1872")
-
-class SkipIfADLS:
-
- # These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_ADLS, reason="SET CACHED not implemented for ADLS")
- hive = pytest.mark.skipif(IS_ADLS, reason="Hive doesn't work with ADLS")
- hdfs_block_size = pytest.mark.skipif(IS_ADLS, reason="ADLS uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_ADLS, reason="HDFS acls are not supported on ADLS")
- jira = partial(pytest.mark.skipif, IS_ADLS)
- hdfs_encryption = pytest.mark.skipif(IS_ADLS,
- reason="HDFS encryption is not supported with ADLS")
-
- # These need test infra work to re-enable.
- udfs = pytest.mark.skipif(IS_ADLS, reason="udas/udfs not copied to ADLS")
- datasrc = pytest.mark.skipif(IS_ADLS, reason="data sources not copied to ADLS")
- hbase = pytest.mark.skipif(IS_ADLS, reason="HBase not started with ADLS")
- qualified_path = pytest.mark.skipif(IS_ADLS,
- reason="Tests rely on HDFS qualified paths, IMPALA-1872")
- eventually_consistent = pytest.mark.skipif(IS_ADLS,
- reason="The client is slow to realize changes to file metadata")
-
-
-class SkipIfGCS:
-
- # These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_GCS, reason="SET CACHED not implemented for GCS")
- hive = pytest.mark.skipif(IS_GCS, reason="Hive doesn't work with GCS")
- hdfs_block_size = pytest.mark.skipif(IS_GCS, reason="GCS uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_GCS, reason="HDFS acls are not supported on GCS")
- jira = partial(pytest.mark.skipif, IS_GCS)
- hdfs_encryption = pytest.mark.skipif(IS_GCS,
- reason="HDFS encryption is not supported with GCS")
-
- # These need test infra work to re-enable.
- hbase = pytest.mark.skipif(IS_GCS, reason="HBase not started with GCS")
- qualified_path = pytest.mark.skipif(IS_GCS,
- reason="Tests rely on HDFS qualified paths, IMPALA-1872")
- variable_listing_times = pytest.mark.skipif(IS_GCS,
- reason="Flakiness due to unpredictable listing times on GCS.")
-
-
-class SkipIfCOS:
-
- # These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_COS, reason="SET CACHED not implemented for COS")
- hive = pytest.mark.skipif(IS_COS, reason="Hive doesn't work with COS")
- hdfs_block_size = pytest.mark.skipif(IS_COS, reason="COS uses it's own block size")
- hdfs_acls = pytest.mark.skipif(IS_COS, reason="HDFS acls are not supported on COS")
- jira = partial(pytest.mark.skipif, IS_COS)
- hdfs_encryption = pytest.mark.skipif(IS_COS,
- reason="HDFS encryption is not supported with COS")
-
- # These need test infra work to re-enable.
- udfs = pytest.mark.skipif(IS_COS, reason="udas/udfs not copied to COS")
- datasrc = pytest.mark.skipif(IS_COS, reason="data sources not copied to COS")
- hbase = pytest.mark.skipif(IS_COS, reason="HBase not started with COS")
- qualified_path = pytest.mark.skipif(IS_COS,
+ hive = pytest.mark.skipif(not IS_HDFS, reason="Hive doesn't work")
+ hbase = pytest.mark.skipif(not IS_HDFS, reason="HBase not started")
+ qualified_path = pytest.mark.skipif(not IS_HDFS,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
- variable_listing_times = pytest.mark.skipif(IS_COS,
- reason="Flakiness due to unpredictable listing times on COS.")
- eventually_consistent = pytest.mark.skipif(IS_COS,
+ variable_listing_times = pytest.mark.skipif(IS_S3 or IS_GCS or IS_COS,
+ reason="Flakiness due to unpredictable listing times on S3.")
+ eventually_consistent = pytest.mark.skipif(IS_ADLS or IS_COS,
reason="The client is slow to realize changes to file metadata")
class SkipIfKudu:
@@ -188,30 +96,10 @@ class SkipIf:
is_buggy_el6_kernel = pytest.mark.skipif(
IS_BUGGY_EL6_KERNEL, reason="Kernel is affected by KUDU-1508")
-
-class SkipIfIsilon:
- caching = pytest.mark.skipif(IS_ISILON, reason="SET CACHED not implemented for Isilon")
- hbase = pytest.mark.skipif(IS_ISILON, reason="HBase not tested with Isilon")
- hive = pytest.mark.skipif(IS_ISILON, reason="Hive not tested with Isilon")
- hdfs_acls = pytest.mark.skipif(IS_ISILON, reason="HDFS acls are not supported on Isilon")
- hdfs_block_size = pytest.mark.skipif(IS_ISILON,
- reason="Isilon uses its own block size")
- hdfs_encryption = pytest.mark.skipif(IS_ISILON,
- reason="HDFS encryption is not supported with Isilon")
- untriaged = pytest.mark.skipif(IS_ISILON,
- reason="This Isilon issue has yet to be triaged.")
- jira = partial(pytest.mark.skipif, IS_ISILON)
-
class SkipIfLocal:
# These are skipped due to product limitations.
- caching = pytest.mark.skipif(IS_LOCAL,
- reason="HDFS caching not supported on local file system")
hdfs_blocks = pytest.mark.skipif(IS_LOCAL,
reason="Files on local filesystem are not split into blocks")
- hdfs_encryption = pytest.mark.skipif(IS_LOCAL,
- reason="HDFS encryption is not supported on local filesystem")
- hive = pytest.mark.skipif(IS_LOCAL,
- reason="Hive not started when using local file system")
multiple_impalad = pytest.mark.skipif(IS_LOCAL,
reason="Multiple impalads are not supported when using local file system")
parquet_file_size = pytest.mark.skipif(IS_LOCAL,
@@ -220,12 +108,8 @@ class SkipIfLocal:
reason="HDFS file handle caching not supported for local non-HDFS files")
# These need test infra work to re-enable.
- hbase = pytest.mark.skipif(IS_LOCAL,
- reason="HBase not started when using local file system")
hdfs_client = pytest.mark.skipif(IS_LOCAL,
reason="HDFS not started when using local file system")
- qualified_path = pytest.mark.skipif(IS_LOCAL,
- reason="Tests rely on HDFS qualified paths")
root_path = pytest.mark.skipif(IS_LOCAL,
reason="Tests rely on the root directory")
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 37da089fe..8fdc8667f 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -35,15 +35,7 @@ from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.resource_pool_config import ResourcePoolConfig
-from tests.common.skip import (
- SkipIfS3,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfABFS,
- SkipIfADLS,
- SkipIfEC,
- SkipIfNotHdfsMinicluster,
- SkipIfOS)
+from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfNotHdfsMinicluster, SkipIfOS
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -444,11 +436,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
assert re.search("Rejected query from pool default-pool: request memory needed "
".* is greater than pool max mem resources 10.00 MB", str(ex))
- @SkipIfS3.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfEC.fix_later
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index e21c02900..2530f68ba 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -24,8 +24,7 @@ import time
from subprocess import check_call
from tests.util.filesystem_utils import get_fs_path
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfOzone,
- SkipIfIsilon, SkipIfGCS, SkipIfCOS, SkipIfLocal)
+from tests.common.skip import SkipIf, SkipIfFS
LOG = logging.getLogger('test_coordinators')
LOG.setLevel(level=logging.DEBUG)
@@ -320,14 +319,7 @@ class TestCoordinators(CustomClusterTestSuite):
num_hosts = "hosts=10 instances=10"
assert num_hosts in str(ret)
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
- @SkipIfIsilon.hbase
- @SkipIfLocal.hbase
+ @SkipIfFS.hbase
@SkipIf.skip_hbase
@pytest.mark.execute_serially
def test_executor_only_hbase(self):
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index 827ba9d28..8fe6ae24a 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -19,20 +19,13 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.util.hive_utils import HiveDbWrapper
from tests.util.event_processor_utils import EventProcessorUtils
from tests.util.filesystem_utils import WAREHOUSE
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfGCS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestEventProcessingCustomConfigs(CustomClusterTestSuite):
"""This class contains tests that exercise the event processing mechanism in the
catalog for non-default configurations"""
diff --git a/tests/custom_cluster/test_hive_parquet_codec_interop.py b/tests/custom_cluster/test_hive_parquet_codec_interop.py
index d03867b9d..c9601d72d 100644
--- a/tests/custom_cluster/test_hive_parquet_codec_interop.py
+++ b/tests/custom_cluster/test_hive_parquet_codec_interop.py
@@ -22,7 +22,7 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.util.event_processor_utils import EventProcessorUtils
from tests.common.environ import HIVE_MAJOR_VERSION
-from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS, SkipIfOzone
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
from tests.util.filesystem_utils import get_fs_path
@@ -52,10 +52,7 @@ class TestParquetInterop(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'parquet')
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("-convert_legacy_hive_parquet_utc_timestamps=true "
"-hdfs_zone_info_zip=%s" % get_fs_path("/test-warehouse/tzdb/2017c.zip"))
diff --git a/tests/custom_cluster/test_hive_text_codec_interop.py b/tests/custom_cluster/test_hive_text_codec_interop.py
index ee41e3f29..a23d88102 100644
--- a/tests/custom_cluster/test_hive_text_codec_interop.py
+++ b/tests/custom_cluster/test_hive_text_codec_interop.py
@@ -21,7 +21,7 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import HIVE_MAJOR_VERSION
-from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS, SkipIfOzone
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
@@ -51,10 +51,7 @@ class TestTextInterop(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'textfile')
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
def test_hive_impala_interop(self, unique_database, cluster_properties):
"""Tests compressed text file written by Hive with different codecs
diff --git a/tests/custom_cluster/test_insert_behaviour.py b/tests/custom_cluster/test_insert_behaviour.py
index 2cd0be2ee..89d06f5cc 100644
--- a/tests/custom_cluster/test_insert_behaviour.py
+++ b/tests/custom_cluster/test_insert_behaviour.py
@@ -18,8 +18,7 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal,
- SkipIfGCS, SkipIfCOS, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfLocal
from tests.util.filesystem_utils import IS_ISILON, WAREHOUSE
from tests.util.hdfs_util import (
HdfsConfig,
@@ -28,12 +27,9 @@ from tests.util.hdfs_util import (
TEST_TBL = "insert_inherit_permission"
-@SkipIfS3.hdfs_acls
-@SkipIfOzone.hdfs_acls
-@SkipIfGCS.hdfs_acls
-@SkipIfCOS.hdfs_acls
-@SkipIfABFS.hdfs_acls
-@SkipIfADLS.hdfs_acls
+
+@SkipIfFS.hdfs_acls
+@SkipIfLocal.hdfs_client
class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
@classmethod
@@ -84,7 +80,6 @@ class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
cls._drop_test_tbl()
super(TestInsertBehaviourCustomCluster, cls).teardown_method(method)
- @SkipIfLocal.hdfs_client
@SkipIfLocal.root_path
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--insert_inherit_permissions=true")
@@ -114,7 +109,6 @@ class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
finally:
client.close()
- @SkipIfLocal.hdfs_client
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--insert_inherit_permissions=false")
def test_insert_inherit_permission_disabled(self):
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 6d08914b7..71668b0db 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -26,8 +26,7 @@ import tempfile
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfGCS, SkipIfCOS,
- SkipIfOzone)
+from tests.common.skip import SkipIfFS
LOG = logging.getLogger(__name__)
@@ -150,12 +149,7 @@ class TestLineage(CustomClusterTestSuite):
lineage_json = json.load(log_file)
assert lineage_json["queryId"] is not profile_query_id
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
+ @SkipIfFS.hbase
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
.format(LINEAGE_TESTS_DIR))
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 1b576f569..53d535ec6 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -27,8 +27,7 @@ import time
from multiprocessing.pool import ThreadPool
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfHive2, SkipIfS3, SkipIfABFS, SkipIfGCS, SkipIfCOS,
- SkipIfADLS, SkipIfIsilon, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfHive2, SkipIfFS
from tests.util.filesystem_utils import WAREHOUSE
RETRY_PROFILE_MSG = 'Retried query planning due to inconsistent metadata'
@@ -535,14 +534,7 @@ class TestFullAcid(CustomClusterTestSuite):
assert res.data == ['0', '1', '2', '3', '4', '5', '6', '7']
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
def test_full_acid_scans(self, vector, unique_database):
self.run_test_case('QueryTest/full-acid-scans', vector, use_db=unique_database)
diff --git a/tests/custom_cluster/test_local_tz_conversion.py b/tests/custom_cluster/test_local_tz_conversion.py
index 1d7635a6b..ca1a720d7 100644
--- a/tests/custom_cluster/test_local_tz_conversion.py
+++ b/tests/custom_cluster/test_local_tz_conversion.py
@@ -17,10 +17,8 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_vector import ImpalaTestDimension
-from tests.common.skip import (SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfOzone, SkipIfGCS,
- SkipIfCOS)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
class TestLocalTzConversion(CustomClusterTestSuite):
@@ -48,12 +46,7 @@ class TestLocalTzConversion(CustomClusterTestSuite):
def get_workload(self):
return 'functional-query'
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
+ @SkipIfFS.hbase
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--use_local_tz_for_unix_timestamp_conversions=true")
def test_timestamp_functions(self, vector):
diff --git a/tests/custom_cluster/test_metadata_no_events_processing.py b/tests/custom_cluster/test_metadata_no_events_processing.py
index 71af95cc5..9d380cf4b 100644
--- a/tests/custom_cluster/test_metadata_no_events_processing.py
+++ b/tests/custom_cluster/test_metadata_no_events_processing.py
@@ -16,17 +16,10 @@
# under the License.
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfGCS,
- SkipIfIsilon, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestMetadataNoEventsProcessing(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args(catalogd_args="--hms_event_polling_interval_s=0")
diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py
index cee6732e8..a458b23aa 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -16,28 +16,12 @@
# under the License.
import pytest
-import re
-from time import sleep
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (
- SkipIfS3,
- SkipIfOzone,
- SkipIfABFS,
- SkipIfADLS,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfIsilon,
- SkipIfLocal)
+from tests.common.skip import SkipIfFS
from tests.util.hive_utils import HiveDbWrapper
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+
+@SkipIfFS.hive
class TestMetadataReplicas(CustomClusterTestSuite):
""" Validates metadata content across catalogd and impalad coordinators."""
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index 63fce015c..48cfa0fce 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -24,8 +24,7 @@ import string
import subprocess
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfGCS,
- SkipIfCOS, SkipIfOzone)
+from tests.common.skip import SkipIfFS
class TestParquetMaxPageHeader(CustomClusterTestSuite):
'''This tests large page headers in parquet files. Parquet page header size can
@@ -101,13 +100,7 @@ class TestParquetMaxPageHeader(CustomClusterTestSuite):
put.stdin.close()
put.wait()
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("-max_page_header_size=31457280")
def test_large_page_header_config(self, vector):
diff --git a/tests/custom_cluster/test_partition.py b/tests/custom_cluster/test_partition.py
index 102a159c5..aa996a5ce 100644
--- a/tests/custom_cluster/test_partition.py
+++ b/tests/custom_cluster/test_partition.py
@@ -20,7 +20,7 @@ import pytest
import shutil
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfIsilon, SkipIfLocal)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -43,8 +43,7 @@ class TestPartition(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--use_local_catalog=true",
@@ -61,8 +60,7 @@ class TestPartition(CustomClusterTestSuite):
except OSError as e:
LOG.info("Cannot remove directory %s, %s " % (local_file_dir, e.strerror))
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--use_local_catalog=false",
diff --git a/tests/custom_cluster/test_permanent_udfs.py b/tests/custom_cluster/test_permanent_udfs.py
index 581e7ba40..3c6719bf6 100644
--- a/tests/custom_cluster/test_permanent_udfs.py
+++ b/tests/custom_cluster/test_permanent_udfs.py
@@ -20,12 +20,10 @@ import os
import pytest
import re
import shutil
-import subprocess
from tempfile import mkdtemp
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfGCS,
- SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.filesystem_utils import get_fs_path
@@ -161,14 +159,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
stmt = "RELOAD FUNCTION ; DESCRIBE FUNCTION {0}.{1}".format(db, udf)
return self.run_stmt_in_hive(stmt)
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
def test_corrupt_java_udf(self):
""" IMPALA-3820: This tests if the Catalog server can gracefully handle
@@ -185,15 +176,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
self.verify_function_count(
"SHOW FUNCTIONS in {0}".format(self.JAVA_FN_TEST_DB), 0)
-
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args= "--local_library_dir={0}".format(LOCAL_LIBRARY_DIR))
@@ -254,14 +237,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
# Make sure we deleted all the temporary jars we copied to the local fs
assert len(glob.glob(self.LOCAL_LIBRARY_DIR + "/*.jar")) == 0
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args= "--local_library_dir={0}".format(LOCAL_LIBRARY_DIR))
@@ -320,14 +296,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
# Make sure we deleted all the temporary jars we copied to the local fs
assert len(glob.glob(self.LOCAL_LIBRARY_DIR + "/*.jar")) == 0
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args= "--local_library_dir={0}".format(LOCAL_LIBRARY_DIR))
diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py
index 23febb8f1..34e83ae92 100644
--- a/tests/custom_cluster/test_query_retries.py
+++ b/tests/custom_cluster/test_query_retries.py
@@ -34,8 +34,8 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.errors import Timeout
-from tests.common.skip import SkipIfEC, SkipIfBuildType, SkipIfGCS, SkipIfCOS
-from tests.common.skip import SkipIfNotHdfsMinicluster
+from tests.common.skip import (SkipIfEC, SkipIfBuildType, SkipIfFS,
+ SkipIfNotHdfsMinicluster)
# The BE krpc port of the impalad to simulate rpc or disk errors in tests.
FAILED_KRPC_PORT = 27001
@@ -292,8 +292,7 @@ class TestQueryRetries(CustomClusterTestSuite):
self.__validate_web_ui_state()
self.__validate_memz()
- @SkipIfGCS.jira(reason="IMPALA-10562")
- @SkipIfCOS.jira(reason="IMPALA-10562")
+ @SkipIfFS.shutdown_idle_fails
@SkipIfBuildType.not_dev_build
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
@@ -359,8 +358,7 @@ class TestQueryRetries(CustomClusterTestSuite):
self.client.close_query(handle)
self.__validate_web_ui_state()
- @SkipIfGCS.jira(reason="IMPALA-10562")
- @SkipIfCOS.jira(reason="IMPALA-10562")
+ @SkipIfFS.shutdown_idle_fails
@SkipIfBuildType.not_dev_build
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index 55b3800c2..a97f7e857 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -35,7 +35,7 @@ from TCLIService import TCLIService
from beeswaxd.BeeswaxService import QueryState
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfGCS, SkipIfCOS
+from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfFS
from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
LOG = logging.getLogger(__name__)
@@ -360,8 +360,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
def get_workload(cls):
return 'functional-query'
- @SkipIfGCS.jira(reason="IMPALA-10562")
- @SkipIfCOS.jira(reason="IMPALA-10562")
+ @SkipIfFS.shutdown_idle_fails
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--shutdown_grace_period_s={grace_period} \
@@ -426,8 +425,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
shutdown_duration = time.time() - start_time
assert shutdown_duration <= self.IDLE_SHUTDOWN_GRACE_PERIOD_S + 10
- @SkipIfGCS.jira(reason="IMPALA-10562")
- @SkipIfCOS.jira(reason="IMPALA-10562")
+ @SkipIfFS.shutdown_idle_fails
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--shutdown_grace_period_s={grace_period} \
diff --git a/tests/custom_cluster/test_rpc_timeout.py b/tests/custom_cluster/test_rpc_timeout.py
index 5a93dbb0c..8b266629f 100644
--- a/tests/custom_cluster/test_rpc_timeout.py
+++ b/tests/custom_cluster/test_rpc_timeout.py
@@ -19,7 +19,7 @@ import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.impala_cluster import ImpalaCluster
-from tests.common.skip import SkipIfBuildType, SkipIfGCS, SkipIfCOS
+from tests.common.skip import SkipIfBuildType, SkipIfFS
from tests.verifiers.metric_verifier import MetricVerifier
# The BE krpc port of the impalad to simulate rpc errors in tests.
@@ -205,8 +205,7 @@ class TestRPCTimeout(CustomClusterTestSuite):
self.execute_query_verify_metrics(self.SLOW_TEST_QUERY,
expected_exception="cancelled due to unresponsive backend")
- @SkipIfGCS.jira(reason="IMPALA-10562")
- @SkipIfCOS.jira(reason="IMPALA-10562")
+ @SkipIfFS.shutdown_idle_fails
@SkipIfBuildType.not_dev_build
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
diff --git a/tests/custom_cluster/test_runtime_profile.py b/tests/custom_cluster/test_runtime_profile.py
index a03385962..3d669c954 100644
--- a/tests/custom_cluster/test_runtime_profile.py
+++ b/tests/custom_cluster/test_runtime_profile.py
@@ -17,7 +17,7 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfEC, SkipIfOzone
+from tests.common.skip import SkipIfEC, SkipIfFS
class TestRuntimeProfile(CustomClusterTestSuite):
@@ -30,7 +30,7 @@ class TestRuntimeProfile(CustomClusterTestSuite):
PERIODIC_COUNTER_UPDATE_FLAG = '--periodic_counter_update_period_ms=50'
# Test depends on block size < 256MiB so larger table is stored in at least 4 blocks.
- @SkipIfOzone.hdfs_block_size
+ @SkipIfFS.large_block_size
@SkipIfEC.different_schedule
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args('--gen_experimental_profile=true ' +
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index 216273538..99d7b659c 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -15,12 +15,10 @@ import pytest
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
+from tests.common.skip import SkipIfFS
-@SkipIfS3.variable_listing_times
-@SkipIfGCS.variable_listing_times
-@SkipIfCOS.variable_listing_times
+@SkipIfFS.variable_listing_times
class TestTopicUpdateFrequency(CustomClusterTestSuite):
@pytest.mark.execute_serially
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index 0b3260f0e..562eef2bf 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -20,13 +20,11 @@
# Tests Impala properly handles errors when reading and writing data.
import pytest
-import random
import subprocess
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfGCS,
- SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIf, SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
class TestDataErrors(ImpalaTestSuite):
@@ -106,12 +104,8 @@ class TestHdfsUnknownErrors(ImpalaTestSuite):
assert error is "", "Couldn't turn Safe mode OFF. Error: %s" % (error)
assert "Safe mode is OFF" in output
-@SkipIfS3.qualified_path
-@SkipIfOzone.qualified_path
-@SkipIfGCS.qualified_path
-@SkipIfCOS.qualified_path
-@SkipIfABFS.qualified_path
-@SkipIfADLS.qualified_path
+
+@SkipIfFS.qualified_path
class TestHdfsScanNodeErrors(TestDataErrors):
@classmethod
def add_test_dimensions(cls):
@@ -128,13 +122,8 @@ class TestHdfsScanNodeErrors(TestDataErrors):
pytest.xfail("Expected results differ across file formats")
self.run_test_case('DataErrorsTest/hdfs-scan-node-errors', vector)
-@SkipIfS3.qualified_path
-@SkipIfOzone.qualified_path
-@SkipIfGCS.qualified_path
-@SkipIfCOS.qualified_path
-@SkipIfABFS.qualified_path
-@SkipIfADLS.qualified_path
-@SkipIfLocal.qualified_path
+
+@SkipIfFS.qualified_path
class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
@classmethod
def add_test_dimensions(cls):
@@ -147,12 +136,7 @@ class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
self.run_test_case('DataErrorsTest/hdfs-sequence-scan-errors', vector)
-@SkipIfS3.qualified_path
-@SkipIfOzone.qualified_path
-@SkipIfGCS.qualified_path
-@SkipIfCOS.qualified_path
-@SkipIfABFS.qualified_path
-@SkipIfADLS.qualified_path
+@SkipIfFS.qualified_path
class TestHdfsRcFileScanNodeErrors(TestHdfsScanNodeErrors):
@classmethod
def add_test_dimensions(cls):
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index 1d6e23891..9283d4919 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -19,16 +19,13 @@
# two types of failures - cancellation of the query and a failure test hook.
#
import pytest
-import os
import re
-from collections import defaultdict
from time import sleep
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
-from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIf, SkipIfFS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.verifiers.metric_verifier import MetricVerifier
@@ -61,15 +58,9 @@ QUERIES = [
WHERE t2.int_col < 1000"""
]
-@SkipIf.skip_hbase # -skip_hbase argument specified
-@SkipIfS3.hbase # S3: missing coverage: failures
-@SkipIfOzone.hbase
-@SkipIfGCS.hbase
-@SkipIfCOS.hbase
-@SkipIfABFS.hbase
-@SkipIfADLS.hbase
-@SkipIfIsilon.hbase # ISILON: missing coverage: failures.
-@SkipIfLocal.hbase
+
+@SkipIf.skip_hbase # -skip_hbase argument specified
+@SkipIfFS.hbase # missing coverage: failures
class TestFailpoints(ImpalaTestSuite):
@classmethod
def get_workload(cls):
diff --git a/tests/metadata/test_catalogd_debug_actions.py b/tests/metadata/test_catalogd_debug_actions.py
index 57d5d5d6c..77cfb58ef 100644
--- a/tests/metadata/test_catalogd_debug_actions.py
+++ b/tests/metadata/test_catalogd_debug_actions.py
@@ -17,12 +17,10 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
+from tests.common.skip import SkipIfFS
-@SkipIfS3.variable_listing_times
-@SkipIfGCS.variable_listing_times
-@SkipIfCOS.variable_listing_times
+@SkipIfFS.variable_listing_times
class TestDebugActions(ImpalaTestSuite):
@pytest.mark.execute_serially
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index 69fa50739..e4c919e42 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -21,9 +21,7 @@ from subprocess import check_call
from tests.common.environ import ImpalaTestClusterProperties
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfCatalogV2,
- SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_single_exec_option_dimension,
@@ -114,14 +112,7 @@ class TestComputeStats(ImpalaTestSuite):
{"compression_codec": c})
self.execute_query_expect_success(self.client, "drop stats {0}".format(table))
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_compute_stats_impala_2201(self, vector, unique_database):
"""IMPALA-2201: Tests that the results of compute incremental stats are properly
persisted when the data was loaded from Hive with hive.stats.autogather=true.
@@ -196,14 +187,7 @@ class TestComputeStats(ImpalaTestSuite):
assert(hdfs_physical_properties_template in explain_result.data[i + 1])
assert("cardinality=0" not in explain_result.data[i + 2])
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_corrupted_stats_in_partitioned_hive_tables(self, vector, unique_database):
"""IMPALA-9744: Tests that the partition stats corruption in Hive tables
(row count=0, partition size>0, persisted when the data was loaded with
@@ -246,14 +230,7 @@ class TestComputeStats(ImpalaTestSuite):
self.create_load_test_corrupt_stats(self, unique_database, create_load_stmts,
table_name, 2, 2)
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_corrupted_stats_in_unpartitioned_hive_tables(self, vector, unique_database):
"""IMPALA-9744: Tests that the stats corruption in unpartitioned Hive
tables (row count=0, partition size>0, persisted when the data was loaded
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 2183a788e..241feebd5 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -29,9 +29,8 @@ from tests.common.environ import (HIVE_MAJOR_VERSION)
from tests.common.file_utils import create_table_from_orc
from tests.common.impala_test_suite import LOG
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import (SkipIf, SkipIfABFS, SkipIfADLS, SkipIfKudu, SkipIfLocal,
- SkipIfCatalogV2, SkipIfHive2, SkipIfS3, SkipIfGCS,
- SkipIfCOS, SkipIfOzone)
+from tests.common.skip import (SkipIf, SkipIfFS, SkipIfKudu, SkipIfLocal,
+ SkipIfCatalogV2, SkipIfHive2)
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import (create_exec_option_dimension,
create_client_protocol_dimension)
@@ -105,7 +104,7 @@ class TestDdlStatements(TestDdlBase):
self.filesystem_client.delete_file_dir(
"test-warehouse/{0}.db/data_t3".format(unique_database), recursive=True)
- @SkipIfADLS.eventually_consistent
+ @SkipIfFS.eventually_consistent
@SkipIfLocal.hdfs_client
def test_drop_cleans_hdfs_dirs(self, unique_database):
self.client.execute('use default')
@@ -153,7 +152,7 @@ class TestDdlStatements(TestDdlBase):
# Re-create database to make unique_database teardown succeed.
self._create_db(unique_database)
- @SkipIfADLS.eventually_consistent
+ @SkipIfFS.eventually_consistent
@SkipIfLocal.hdfs_client
def test_truncate_cleans_hdfs_files(self, unique_database):
# Verify the db directory exists
@@ -305,10 +304,7 @@ class TestDdlStatements(TestDdlBase):
use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
@SkipIfHive2.orc
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@UniqueDatabase.parametrize(sync_ddl=True)
def test_create_table_like_file_orc(self, vector, unique_database):
COMPLEXTYPETBL_PATH = 'test-warehouse/managed/functional_orc_def.db/' \
diff --git a/tests/metadata/test_event_processing.py b/tests/metadata/test_event_processing.py
index eeac52914..dfe98ea3d 100644
--- a/tests/metadata/test_event_processing.py
+++ b/tests/metadata/test_event_processing.py
@@ -14,20 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from tests.common.skip import SkipIfHive2, SkipIfCatalogV2
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfHive2, SkipIfCatalogV2
from tests.util.event_processor_utils import EventProcessorUtils
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfGCS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
@SkipIfCatalogV2.hms_event_polling_disabled()
class TestEventProcessing(ImpalaTestSuite):
"""This class contains tests that exercise the event processing mechanism in the
diff --git a/tests/metadata/test_hdfs_encryption.py b/tests/metadata/test_hdfs_encryption.py
index 06724d120..897180fd5 100644
--- a/tests/metadata/test_hdfs_encryption.py
+++ b/tests/metadata/test_hdfs_encryption.py
@@ -19,8 +19,7 @@ import getpass
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -34,14 +33,7 @@ PYWEBHDFS_TMP_DIR = 'tmp/test_encryption_load_data'
TMP_DIR = '/%s' % (PYWEBHDFS_TMP_DIR)
-@SkipIfS3.hdfs_encryption
-@SkipIfOzone.hdfs_encryption
-@SkipIfGCS.hdfs_encryption
-@SkipIfCOS.hdfs_encryption
-@SkipIfABFS.hdfs_encryption
-@SkipIfADLS.hdfs_encryption
-@SkipIfIsilon.hdfs_encryption
-@SkipIfLocal.hdfs_encryption
+@SkipIfFS.hdfs_encryption
@pytest.mark.execute_serially
class TestHdfsEncryption(ImpalaTestSuite):
''' Tests LOAD DATA commands work between HDFS encryption zones.
@@ -151,7 +143,6 @@ class TestHdfsEncryption(ImpalaTestSuite):
else:
self.client.execute('load data inpath \'%s\' into table tbl ' % (TMP_DIR))
- @SkipIfIsilon.hdfs_encryption
@pytest.mark.execute_serially
def test_drop_partition_encrypt(self):
"""Verifies if alter <tbl> drop partition purge works in case
@@ -222,7 +213,6 @@ class TestHdfsEncryption(ImpalaTestSuite):
assert not self.hdfs_client.exists("test-warehouse/{0}.db/t1/j=3/j3.txt".format(TEST_DB))
assert not self.hdfs_client.exists("test-warehouse/{0}.db/t1/j=3".format(TEST_DB))
- @SkipIfIsilon.hdfs_encryption
@pytest.mark.execute_serially
def test_drop_table_encrypt(self):
"""Verifies if drop <table> purge works in a case where Trash directory and table
diff --git a/tests/metadata/test_hdfs_permissions.py b/tests/metadata/test_hdfs_permissions.py
index d0e74e368..2ff4e1e6a 100644
--- a/tests/metadata/test_hdfs_permissions.py
+++ b/tests/metadata/test_hdfs_permissions.py
@@ -16,8 +16,7 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal,
- SkipIfGCS, SkipIfCOS, SkipIfCatalogV2, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -28,12 +27,7 @@ TEST_TBL = 'read_only_tbl'
TBL_LOC = '%s/%s' % (WAREHOUSE, TEST_TBL)
-@SkipIfS3.hdfs_acls
-@SkipIfOzone.hdfs_acls
-@SkipIfGCS.hdfs_acls
-@SkipIfCOS.hdfs_acls
-@SkipIfABFS.hdfs_acls
-@SkipIfADLS.hdfs_acls
+@SkipIfFS.hdfs_acls
@SkipIfLocal.hdfs_client
class TestHdfsPermissions(ImpalaTestSuite):
@classmethod
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index e303c626b..cea6101fc 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -27,13 +27,10 @@
import pytest
import random
import string
-from subprocess import call
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfHive2, SkipIfHive3,
- SkipIfIsilon, SkipIfGCS, SkipIfCOS, SkipIfLocal,
- SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfHive2, SkipIfHive3
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -41,14 +38,7 @@ from tests.util.event_processor_utils import EventProcessorUtils
from tests.util.hive_utils import HiveDbWrapper, HiveTableWrapper
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestHmsIntegrationSanity(ImpalaTestSuite):
@classmethod
def get_workload(self):
@@ -153,14 +143,8 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
else:
assert False
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+
+@SkipIfFS.hive
class TestHmsIntegration(ImpalaTestSuite):
@classmethod
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index ed2f54302..357cc8bc8 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -20,11 +20,8 @@
import pytest
import re
-from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfIsilon, SkipIfS3, SkipIfABFS, SkipIfADLS,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfCatalogV2,
- SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfCatalogV2
from tests.common.test_dimensions import ALL_NODES_ONLY
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -76,14 +73,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
# Missing Coverage: Describe formatted compatibility between Impala and Hive when the
# data doesn't reside in hdfs.
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_describe_formatted(self, vector, unique_database):
# IMPALA-10176: test_describe_formatted is broken, so disable it for now
pytest.skip()
@@ -171,14 +161,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
for name in self.TEST_DATA_SRC_NAMES:
self.client.execute(self.CREATE_DATA_SRC_STMT % (name,))
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially # because of use of hardcoded database
def test_describe_db(self, vector, cluster_properties):
self.__test_describe_db_cleanup()
diff --git a/tests/metadata/test_partition_metadata.py b/tests/metadata/test_partition_metadata.py
index b68e2cd91..0dd913182 100644
--- a/tests/metadata/test_partition_metadata.py
+++ b/tests/metadata/test_partition_metadata.py
@@ -17,11 +17,10 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfLocal
from tests.common.test_dimensions import (create_single_exec_option_dimension,
create_uncompressed_text_dimension)
-from tests.util.filesystem_utils import get_fs_path, WAREHOUSE, FILESYSTEM_PREFIX
+from tests.util.filesystem_utils import WAREHOUSE, FILESYSTEM_PREFIX
# Map from the test dimension file_format string to the SQL "STORED AS"
# argument.
@@ -89,14 +88,7 @@ class TestPartitionMetadata(ImpalaTestSuite):
data = self.execute_scalar("select sum(i), sum(j) from %s" % FQ_TBL_NAME)
assert data.split('\t') == ['6', '9']
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_partition_metadata_compatibility(self, vector, unique_database):
"""Regression test for IMPALA-2048. For partitioned tables, test that when Impala
updates the partition metadata (e.g. by doing a compute stats), the tables are
diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py
index 5d8d73621..d2319f226 100644
--- a/tests/metadata/test_recover_partitions.py
+++ b/tests/metadata/test_recover_partitions.py
@@ -20,7 +20,7 @@
import os
from six.moves import urllib
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfLocal, SkipIfS3, SkipIfCatalogV2
+from tests.common.skip import SkipIfLocal, SkipIfFS, SkipIfCatalogV2
from tests.common.test_dimensions import ALL_NODES_ONLY
from tests.common.test_dimensions import create_exec_option_dimension
from tests.util.filesystem_utils import WAREHOUSE, IS_S3
@@ -378,7 +378,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
self.verify_partitions(parts, result.data)
@SkipIfLocal.hdfs_client
- @SkipIfS3.empty_directory
+ @SkipIfFS.empty_directory
def test_empty_directory(self, vector, unique_database):
"""Explicitly test how empty directories are handled when partitions are recovered."""
diff --git a/tests/metadata/test_recursive_listing.py b/tests/metadata/test_recursive_listing.py
index d2d50f261..321b78a2d 100644
--- a/tests/metadata/test_recursive_listing.py
+++ b/tests/metadata/test_recursive_listing.py
@@ -17,8 +17,7 @@ import time
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.test_dimensions import create_uncompressed_text_dimension
-from tests.common.skip import (SkipIfLocal, SkipIfS3, SkipIfGCS, SkipIfCOS,
- SkipIfADLS)
+from tests.common.skip import SkipIfLocal, SkipIfFS
from tests.util.filesystem_utils import WAREHOUSE
@@ -142,10 +141,8 @@ class TestRecursiveListing(ImpalaTestSuite):
assert len(self._show_files(fq_tbl_name)) == 1
assert len(self._get_rows(fq_tbl_name)) == 1
- @SkipIfS3.variable_listing_times
- @SkipIfCOS.variable_listing_times
- @SkipIfGCS.variable_listing_times
- @SkipIfADLS.eventually_consistent
+ @SkipIfFS.variable_listing_times
+ @SkipIfFS.eventually_consistent
@pytest.mark.execute_serially
@pytest.mark.stress
def test_large_staging_dirs(self, unique_database):
@@ -169,10 +166,8 @@ class TestRecursiveListing(ImpalaTestSuite):
pause_ms_before_file_cleanup=300,
refresh_should_fail=False)
- @SkipIfS3.variable_listing_times
- @SkipIfCOS.variable_listing_times
- @SkipIfGCS.variable_listing_times
- @SkipIfADLS.eventually_consistent
+ @SkipIfFS.variable_listing_times
+ @SkipIfFS.eventually_consistent
@pytest.mark.execute_serially
@pytest.mark.stress
def test_partition_dir_removed_inflight(self, unique_database):
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index cf2c63474..984f96ebb 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -17,19 +17,11 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.util.filesystem_utils import get_fs_path
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestRefreshPartition(ImpalaTestSuite):
"""
This class tests the functionality to refresh a partition individually
diff --git a/tests/metadata/test_stale_metadata.py b/tests/metadata/test_stale_metadata.py
index e664b97e2..9611939fd 100644
--- a/tests/metadata/test_stale_metadata.py
+++ b/tests/metadata/test_stale_metadata.py
@@ -19,7 +19,7 @@ from subprocess import check_call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
@@ -76,8 +76,7 @@ class TestRewrittenFile(ImpalaTestSuite):
result = self.client.execute("select count(*) from %s.%s" % (db_name, table_name))
assert result.data == [str(expected_new_count)]
- @SkipIfS3.jira(reason="IMPALA-2512")
- @SkipIfGCS.jira(reason="IMPALA-2512")
+ @SkipIfFS.read_past_eof
def test_new_file_shorter(self, vector, unique_database):
"""Rewrites an existing file with a new shorter file."""
# Full error is something like:
diff --git a/tests/metadata/test_stats_extrapolation.py b/tests/metadata/test_stats_extrapolation.py
index 8add7ebe8..5f613a495 100644
--- a/tests/metadata/test_stats_extrapolation.py
+++ b/tests/metadata/test_stats_extrapolation.py
@@ -17,12 +17,12 @@
from os import path
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfEC, SkipIfOzone
+from tests.common.skip import SkipIfEC, SkipIfFS
from tests.common.test_dimensions import (
- create_exec_option_dimension,
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
+
class TestStatsExtrapolation(ImpalaTestSuite):
"""Test stats extrapolation and compute stats tablesample. Stats extrapolation is
enabled via table property and not via the impalad startup flag so these tests can be
@@ -39,7 +39,7 @@ class TestStatsExtrapolation(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
- @SkipIfOzone.no_storage_ids
+ @SkipIfFS.no_storage_ids
@SkipIfEC.contain_full_explain
def test_stats_extrapolation(self, vector, unique_database):
vector.get_value('exec_option')['num_nodes'] = 1
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index 69a57acad..7daedc711 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-import os
import pprint
import pytest
import shlex
@@ -24,8 +23,7 @@ from subprocess import call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader
@@ -46,16 +44,9 @@ from tests.util.test_file_parser import QueryTestSectionReader
# in Impala and Hive would be insufficient.
-# Missing Coverage: Views created by Hive and Impala being visible and queryble by each
+# Missing Coverage: Views created by Hive and Impala being visible and queryable by each
# other on non hdfs storage.
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestViewCompatibility(ImpalaTestSuite):
VALID_SECTION_NAMES = ["CREATE_VIEW", "CREATE_VIEW_RESULTS",\
"QUERY_HIVE_VIEW_RESULTS", "QUERY_IMPALA_VIEW_RESULTS"]
diff --git a/tests/query_test/test_acid.py b/tests/query_test/test_acid.py
index 1335e1454..d3d3ed17f 100644
--- a/tests/query_test/test_acid.py
+++ b/tests/query_test/test_acid.py
@@ -24,9 +24,7 @@ import time
from hive_metastore.ttypes import CommitTxnRequest, LockType, OpenTxnRequest
from subprocess import check_call
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfS3, SkipIfABFS,
- SkipIfADLS, SkipIfIsilon, SkipIfGCS, SkipIfCOS,
- SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfFS
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.acid_txn import AcidTxn
@@ -46,14 +44,7 @@ class TestAcid(ImpalaTestSuite):
v.get_value('table_format').file_format in ['text'])
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_basic(self, vector, unique_database):
self.run_test_case('QueryTest/acid', vector, use_db=unique_database)
@@ -66,64 +57,29 @@ class TestAcid(ImpalaTestSuite):
self.run_test_case('QueryTest/acid-no-hive', vector, use_db=unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_compaction(self, vector, unique_database):
self.run_test_case('QueryTest/acid-compaction', vector, use_db=unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_negative(self, vector, unique_database):
self.run_test_case('QueryTest/acid-negative', vector, use_db=unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_truncate(self, vector, unique_database):
self.run_test_case('QueryTest/acid-truncate', vector, use_db=unique_database)
assert "0" == self.run_stmt_in_hive("select count(*) from {0}.{1}".format(
unique_database, "pt")).split("\n")[1]
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_partitioned(self, vector, unique_database):
self.run_test_case('QueryTest/acid-partitioned', vector, use_db=unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_full_acid_scans(self, vector, unique_database):
self.run_test_case('QueryTest/full-acid-scans', vector, use_db=unique_database)
@@ -138,14 +94,7 @@ class TestAcid(ImpalaTestSuite):
# it can not be shown in the query profile. Skip CatalogV2 to avoid flaky tests.
@SkipIfHive2.acid
@SkipIfCatalogV2.hms_event_polling_enabled()
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_profile(self, vector, unique_database):
self.run_test_case('QueryTest/acid-profile', vector, use_db=unique_database)
@@ -154,14 +103,7 @@ class TestAcid(ImpalaTestSuite):
self.run_test_case('QueryTest/full-acid-rowid', vector, use_db=unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_full_acid_original_files(self, vector, unique_database):
table_name = "alltypes_promoted_nopart"
fq_table_name = "{0}.{1}".format(unique_database, table_name)
@@ -179,14 +121,7 @@ class TestAcid(ImpalaTestSuite):
self.run_test_case('QueryTest/full-acid-original-file', vector, unique_database)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_insert_statschg(self, vector, unique_database):
self.run_test_case('QueryTest/acid-clear-statsaccurate',
vector, use_db=unique_database)
@@ -198,14 +133,7 @@ class TestAcid(ImpalaTestSuite):
.format(unique_database, "insertonly_part_colstats"))
assert "2" in result
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_ext_statschg(self, vector, unique_database):
self.run_test_case('QueryTest/clear-statsaccurate',
vector, use_db=unique_database)
@@ -219,14 +147,7 @@ class TestAcid(ImpalaTestSuite):
assert "2" in result
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_compute_stats(self, vector, unique_database):
self.run_test_case('QueryTest/acid-compute-stats', vector, use_db=unique_database)
@@ -236,14 +157,7 @@ class TestAcid(ImpalaTestSuite):
# Negative test for LOAD DATA INPATH and all other SQL that we don't support.
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_acid_heartbeats(self, vector, unique_database):
"""Tests heartbeating of transactions. Creates a long-running query via
some jitting and in the meanwhile it periodically checks whether there is
@@ -316,14 +230,7 @@ class TestAcid(ImpalaTestSuite):
commit_req.txnid = txn_id
return self.hive_client.commit_txn(commit_req)
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_lock_timings(self, vector, unique_database):
def elapsed_time_for_query(query):
t_start = time.time()
@@ -364,14 +271,7 @@ class TestAcid(ImpalaTestSuite):
acid_util.unlock(lock_resp.lockid)
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_in_progress_compactions(self, vector, unique_database):
"""Checks that in-progress compactions are not visible. The test mimics
in-progress compactions by opening a transaction and creating a new base
diff --git a/tests/query_test/test_date_queries.py b/tests/query_test/test_date_queries.py
index ae64cdd88..56e37aeb2 100644
--- a/tests/query_test/test_date_queries.py
+++ b/tests/query_test/test_date_queries.py
@@ -20,8 +20,7 @@
import pytest
from tests.common.file_utils import create_table_and_copy_files
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal, SkipIfGCS,
- SkipIfCOS, SkipIfOzone)
+from tests.common.skip import SkipIfFS
from tests.common.test_dimensions import (create_exec_option_dimension_from_dict,
create_client_protocol_dimension, hs2_parquet_constraint)
from tests.shell.util import create_impala_shell_executable_dimension
@@ -71,13 +70,7 @@ class TestDateQueries(ImpalaTestSuite):
pytest.skip()
self.run_test_case('QueryTest/date-partitioning', vector, use_db=unique_database)
- @SkipIfS3.qualified_path
- @SkipIfOzone.qualified_path
- @SkipIfGCS.qualified_path
- @SkipIfCOS.qualified_path
- @SkipIfABFS.qualified_path
- @SkipIfADLS.qualified_path
- @SkipIfLocal.qualified_path
+ @SkipIfFS.qualified_path
def test_fileformat_support(self, vector, unique_database):
""" Test that scanning and writing DATE is supported for text and parquet tables.
Test that scanning DATE is supported for avro tables as well.
diff --git a/tests/query_test/test_hbase_queries.py b/tests/query_test/test_hbase_queries.py
index f92d1b057..a1acbf33b 100644
--- a/tests/query_test/test_hbase_queries.py
+++ b/tests/query_test/test_hbase_queries.py
@@ -19,17 +19,8 @@
import pytest
-from tests.common.skip import (
- SkipIfIsilon,
- SkipIfS3,
- SkipIfOzone,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfABFS,
- SkipIfADLS,
- SkipIfLocal)
-
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfFS
class TestHBaseQueries(ImpalaTestSuite):
@classmethod
@@ -67,14 +58,7 @@ class TestHBaseQueries(ImpalaTestSuite):
def test_hbase_inserts(self, vector):
self.run_test_case('QueryTest/hbase-inserts', vector)
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_hbase_col_filter(self, vector, unique_database):
"""IMPALA-7929: test query with table created with hive and mapped to hbase. The key
column doesn't have qualifier and the query with predicate on key column name should
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 666829a63..0d65ab4da 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -25,22 +25,14 @@ from subprocess import check_call
from tests.common.environ import build_flavor_timeout, IS_DOCKERIZED_TEST_CLUSTER
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfEC,
- SkipIfDockerizedCluster, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfDockerizedCluster
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
from tests.util.shell_util import exec_process
+
# End to end test that hdfs caching is working.
-@SkipIfS3.caching # S3: missing coverage: verify SET CACHED gives error
-@SkipIfOzone.caching
-@SkipIfGCS.caching
-@SkipIfCOS.caching
-@SkipIfABFS.caching
-@SkipIfADLS.caching
-@SkipIfIsilon.caching
-@SkipIfLocal.caching
+@SkipIfFS.hdfs_caching # missing coverage: verify SET CACHED gives error
@SkipIfEC.fix_later
class TestHdfsCaching(ImpalaTestSuite):
@classmethod
@@ -113,30 +105,17 @@ class TestHdfsCaching(ImpalaTestSuite):
result = self.execute_query(query_string)
assert(len(result.data) == 2)
+
# A separate class has been created for "test_hdfs_caching_fallback_path" to make it
# run as a part of exhaustive tests which require the workload to be 'functional-query'.
# TODO: Move this to TestHdfsCaching once we make exhaustive tests run for other workloads
-@SkipIfS3.caching
-@SkipIfOzone.caching
-@SkipIfGCS.caching
-@SkipIfCOS.caching
-@SkipIfABFS.caching
-@SkipIfADLS.caching
-@SkipIfIsilon.caching
-@SkipIfLocal.caching
+@SkipIfFS.hdfs_caching
class TestHdfsCachingFallbackPath(ImpalaTestSuite):
@classmethod
def get_workload(self):
return 'functional-query'
- @SkipIfS3.hdfs_encryption
- @SkipIfOzone.hdfs_encryption
- @SkipIfGCS.hdfs_encryption
- @SkipIfCOS.hdfs_encryption
- @SkipIfABFS.hdfs_encryption
- @SkipIfADLS.hdfs_encryption
- @SkipIfIsilon.hdfs_encryption
- @SkipIfLocal.hdfs_encryption
+ @SkipIfFS.hdfs_encryption
def test_hdfs_caching_fallback_path(self, vector, unique_database, testid_checksum):
""" This tests the code path of the query execution where the hdfs cache read fails
and the execution falls back to the normal read path. To reproduce this situation we
@@ -184,14 +163,7 @@ class TestHdfsCachingFallbackPath(ImpalaTestSuite):
shell=False)
-@SkipIfS3.caching
-@SkipIfOzone.caching
-@SkipIfGCS.caching
-@SkipIfCOS.caching
-@SkipIfABFS.caching
-@SkipIfADLS.caching
-@SkipIfIsilon.caching
-@SkipIfLocal.caching
+@SkipIfFS.hdfs_caching
class TestHdfsCachingDdl(ImpalaTestSuite):
@classmethod
def get_workload(self):
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 30d75fdc3..b57308add 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -25,8 +25,8 @@ from testdata.common import widetable
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, \
- SkipIfHive2, SkipIfNotHdfsMinicluster, SkipIfS3, SkipIfDockerizedCluster
+from tests.common.skip import (SkipIfFS, SkipIfEC, SkipIfLocal, SkipIfHive2,
+ SkipIfNotHdfsMinicluster)
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension,
@@ -136,7 +136,7 @@ class TestInsertQueries(ImpalaTestSuite):
# Erasure coding doesn't respect memory limit
@SkipIfEC.fix_later
# ABFS partition names cannot end in periods
- @SkipIfABFS.file_or_folder_name_ends_with_period
+ @SkipIfFS.file_or_folder_name_ends_with_period
def test_insert(self, vector, unique_database):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index bef5a6ef5..0e6580141 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -23,11 +23,11 @@ import re
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfDockerizedCluster,
- SkipIfCatalogV2, SkipIfOzone)
+from tests.common.skip import (SkipIfFS, SkipIfLocal, SkipIfDockerizedCluster,
+ SkipIfCatalogV2)
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_S3
+
@SkipIfLocal.hdfs_client
class TestInsertBehaviour(ImpalaTestSuite):
"""Tests for INSERT behaviour that isn't covered by checking query results"""
@@ -48,7 +48,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
if method.__name__ == "test_insert_select_with_empty_resultset":
self.cleanup_db(self.TEST_DB_NAME)
- @SkipIfADLS.eventually_consistent
+ @SkipIfFS.eventually_consistent
@pytest.mark.execute_serially
def test_insert_removes_staging_files(self):
TBL_NAME = "insert_overwrite_nopart"
@@ -133,13 +133,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
# a subdirectory)
assert len(self.filesystem_client.ls(part_dir)) == 1
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@pytest.mark.xfail(run=False, reason="Fails intermittently on test clusters")
@pytest.mark.execute_serially
def test_insert_inherit_acls(self):
@@ -198,13 +192,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
"PARTITION(p1=1, p2=2, p3=30) VALUES(1)")
check_has_acls("p1=1/p2=2/p3=30", "default:group:new_leaf_group:-w-")
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfCatalogV2.impala_7539()
def test_insert_file_permissions(self, unique_database):
"""Test that INSERT correctly respects file permission (minimum ACLs)"""
@@ -254,13 +242,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
# Should be writable because 'other' ACLs allow writes
self.execute_query_expect_success(self.client, insert_query)
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfCatalogV2.impala_7539()
def test_mixed_partition_permissions(self, unique_database):
"""
@@ -339,13 +321,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.execute_query_expect_success(self.client, insert_query("added_part"))
load_data(self.execute_query_expect_success, "added_part")
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfCatalogV2.impala_7539()
def test_readonly_table_dir(self, unique_database):
"""
@@ -375,13 +351,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
err = self.execute_query_expect_failure(self.client, insert_query("some_new_part"))
assert re.search(r'Impala does not have WRITE access.*' + table_path, str(err))
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfDockerizedCluster.insert_acls
@SkipIfCatalogV2.impala_7539()
def test_insert_acl_permissions(self, unique_database):
@@ -459,13 +429,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
# Should be writable because 'other' ACLs allow writes
self.execute_query_expect_success(self.client, insert_query)
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfCatalogV2.impala_7539()
def test_load_permissions(self, unique_database):
# We rely on test_insert_acl_permissions() to exhaustively check that ACL semantics
@@ -518,8 +482,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
# We expect this to succeed, it's not an error if all files in the dir cannot be read
self.execute_query_expect_success(self.client, load_dir_query)
- @SkipIfADLS.eventually_consistent
- @SkipIfCOS.eventually_consistent
+ @SkipIfFS.eventually_consistent
@pytest.mark.execute_serially
def test_insert_select_with_empty_resultset(self):
"""Test insert/select query won't trigger partition directory or zero size data file
@@ -589,13 +552,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.filesystem_client.delete_file_dir(target_table_path, recursive=True)
self.execute_query_expect_failure(self.client, insert_query)
- @SkipIfS3.hdfs_acls
- @SkipIfOzone.hdfs_acls
- @SkipIfGCS.hdfs_acls
- @SkipIfCOS.hdfs_acls
- @SkipIfABFS.hdfs_acls
- @SkipIfADLS.hdfs_acls
- @SkipIfIsilon.hdfs_acls
+ @SkipIfFS.hdfs_acls
@SkipIfDockerizedCluster.insert_acls
@SkipIfCatalogV2.impala_7539()
def test_multiple_group_acls(self, unique_database):
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 8bd71757d..563d090ef 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -20,7 +20,7 @@
import os
from collections import namedtuple
-from datetime import (datetime, date)
+from datetime import datetime, date
from decimal import Decimal
from subprocess import check_call
from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder, ConvertedType
@@ -28,8 +28,7 @@ from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder, Convert
from tests.common.environ import impalad_basedir
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import (SkipIfEC, SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfABFS,
- SkipIfADLS, SkipIfGCS, SkipIfCOS, SkipIfOzone)
+from tests.common.skip import SkipIfEC, SkipIfFS, SkipIfLocal
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
from tests.common.test_vector import ImpalaTestDimension
@@ -200,7 +199,7 @@ class TestInsertParquetVerifySize(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension("compression_codec", *PARQUET_CODECS))
- @SkipIfIsilon.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfLocal.hdfs_client
def test_insert_parquet_verify_size(self, vector, unique_database):
# Test to verify that the result file size is close to what we expect.
@@ -535,14 +534,7 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
# by python to string. In both HS2 and beewax, it only handles float
# precision uptil 16 decimal digits and test needs 17.
# IMPALA-9365 describes why HS2 is not started on non-HDFS test env.
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_double_precision(self, vector, unique_database):
# IMPALA-10654: Test inserting double into Parquet table retains the precision.
src_tbl = "{0}.{1}".format(unique_database, "i10654_parquet")
@@ -556,14 +548,8 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
result = self.run_stmt_in_hive(select_stmt)
assert result.split('\n')[1] == '-0.43149576573887316'
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
+
+@SkipIfFS.hive
# TODO: Should we move this to test_parquet_stats.py?
class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py
index 29993f5d9..8961b6391 100644
--- a/tests/query_test/test_join_queries.py
+++ b/tests/query_test/test_join_queries.py
@@ -21,16 +21,7 @@ import pytest
from copy import deepcopy
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (
- SkipIf,
- SkipIfIsilon,
- SkipIfLocal,
- SkipIfS3,
- SkipIfOzone,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfABFS,
- SkipIfADLS)
+from tests.common.skip import SkipIf, SkipIfFS
from tests.common.test_vector import ImpalaTestDimension
class TestJoinQueries(ImpalaTestSuite):
@@ -81,15 +72,8 @@ class TestJoinQueries(ImpalaTestSuite):
del new_vector.get_value('exec_option')['batch_size'] # .test file sets batch_size
self.run_test_case('QueryTest/single-node-joins-with-limits-exhaustive', new_vector)
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
- @SkipIfIsilon.hbase
+ @SkipIfFS.hbase
@SkipIf.skip_hbase
- @SkipIfLocal.hbase
def test_joins_against_hbase(self, vector):
new_vector = deepcopy(vector)
new_vector.get_value('exec_option')['batch_size'] = vector.get_value('batch_size')
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index 4f5b50d3a..4c9421046 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -25,7 +25,7 @@ from tests.common.environ import ImpalaTestClusterProperties, build_flavor_timeo
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.kudu_test_suite import KuduTestSuite
-from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfNotHdfsMinicluster
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import IS_HDFS
@@ -130,7 +130,7 @@ class TestMtDopParquet(ImpalaTestSuite):
self.run_test_case('QueryTest/parquet-filtering', vector)
@pytest.mark.execute_serially
- @SkipIfABFS.file_or_folder_name_ends_with_period
+ @SkipIfFS.file_or_folder_name_ends_with_period
def test_mt_dop_insert(self, vector, unique_database):
"""Basic tests for inserts with mt_dop > 0"""
mt_dop = vector.get_value('mt_dop')
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index d28314d26..0097fa117 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -18,24 +18,10 @@
import os
from copy import deepcopy
import pytest
-from subprocess import check_call
-from pytest import skip
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (
- SkipIfIsilon,
- SkipIfS3,
- SkipIfOzone,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfABFS,
- SkipIfADLS,
- SkipIfEC,
- SkipIfHive2,
- SkipIfLocal,
- SkipIfNotHdfsMinicluster
- )
+from tests.common.skip import SkipIfFS, SkipIfEC, SkipIfHive2, SkipIfNotHdfsMinicluster
from tests.common.test_dimensions import (create_exec_option_dimension,
create_exec_option_dimension_from_dict, create_client_protocol_dimension,
create_orc_dimension, orc_schema_resolution_constraint)
@@ -317,14 +303,7 @@ class TestNestedTypesNoMtDop(ImpalaTestSuite):
pytest.skip('This test is specific to Parquet')
self.run_test_case('QueryTest/nested-types-parquet-stats', vector)
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_upper_case_field_name(self, unique_database):
"""IMPALA-5994: Tests that a Hive-created table with a struct field name with upper
case characters can be selected."""
@@ -375,14 +354,7 @@ class TestNestedTypesNoMtDop(ImpalaTestSuite):
# Skip this test on non-HDFS filesystems, because the test contains Hive
# queries that hang in some cases due to IMPALA-9365.
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@SkipIfHive2.acid
def test_partitioned_table_acid(self, vector, unique_database):
"""IMPALA-6370: Test that a partitioned table with nested types can be scanned."""
@@ -801,7 +773,7 @@ class TestParquetArrayEncodings(ImpalaTestSuite):
# The Parquet resolution policy is manually set in the .test files.
if vector.get_value('parquet_array_resolution') != "three_level":
- skip("Test only run with three_level")
+ pytest.skip("Test only run with three_level")
ambig_modern_tbl = "ambig_modern"
self._create_test_table(unique_database, ambig_modern_tbl,
@@ -884,14 +856,7 @@ class TestMaxNestingDepth(ImpalaTestSuite):
self.run_stmt_in_hive(insert_table)
self.client.execute("INVALIDATE METADATA %s" % tbl_name)
- @SkipIfIsilon.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_load_hive_table(self, vector, unique_database):
"""Tests that Impala rejects Hive-created tables with complex types that exceed
the maximum nesting depth."""
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index aae450604..a5721b78f 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -19,9 +19,7 @@ from collections import defaultdict
from datetime import datetime
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone,
- SkipIfNotHdfsMinicluster)
+from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfNotHdfsMinicluster
from tests.util.filesystem_utils import IS_EC
from time import sleep
from RuntimeProfile.ttypes import TRuntimeProfileFormat
@@ -92,14 +90,7 @@ class TestObservability(ImpalaTestSuite):
assert num_validated > 0
self.hs2_client.close_query(handle)
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
- @SkipIfLocal.hbase
- @SkipIfIsilon.hbase
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
+ @SkipIfFS.hbase
def test_scan_summary(self):
"""IMPALA-4499: Checks that the exec summary for scans show the table name."""
# HDFS table
@@ -676,14 +667,7 @@ class TestObservability(ImpalaTestSuite):
self.__check_query_profile_storage_load_time(unique_database, table_name,
cluster_properties)
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
- @SkipIfLocal.hbase
- @SkipIfIsilon.hbase
- @SkipIfABFS.hbase
- @SkipIfADLS.hbase
+ @SkipIfFS.hbase
@pytest.mark.execute_serially
def test_query_profile_storage_load_time(self, cluster_properties):
"""Test that when a query needs load metadata for table(s), the
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 4bcc792de..acabdcbbc 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -15,13 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-import os
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
# Tests to validate HDFS partitioning.
@@ -47,14 +45,7 @@ class TestPartitioning(ImpalaTestSuite):
# Missing Coverage: Impala deals with boolean partitions created by Hive on a non-hdfs
# filesystem.
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
+ @SkipIfFS.hive
def test_boolean_partitions(self, vector, unique_database):
# This test takes about a minute to complete due to the Hive commands that are
# executed. To cut down on runtime, limit the test to exhaustive exploration
diff --git a/tests/query_test/test_resource_limits.py b/tests/query_test/test_resource_limits.py
index 9b0d0d347..37081e415 100644
--- a/tests/query_test/test_resource_limits.py
+++ b/tests/query_test/test_resource_limits.py
@@ -16,8 +16,7 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfEC, SkipIfLocal, SkipIfS3, SkipIfABFS,
- SkipIfGCS, SkipIfCOS, SkipIfADLS, SkipIfOzone)
+from tests.common.skip import SkipIfEC, SkipIfLocal, SkipIfFS
from tests.common.test_dimensions import create_parquet_dimension
@@ -45,12 +44,7 @@ class TestResourceLimits(ImpalaTestSuite):
def test_resource_limits(self, vector):
self.run_test_case('QueryTest/query-resource-limits', vector)
- @SkipIfS3.hbase
- @SkipIfOzone.hbase
- @SkipIfGCS.hbase
- @SkipIfCOS.hbase
- @SkipIfADLS.hbase
- @SkipIfABFS.hbase
+ @SkipIfFS.hbase
@SkipIfLocal.multiple_impalad
def test_resource_limits_hbase(self, vector):
self.run_test_case('QueryTest/query-resource-limits-hbase', vector)
diff --git a/tests/query_test/test_runtime_filters.py b/tests/query_test/test_runtime_filters.py
index 089de24ca..da5913c0b 100644
--- a/tests/query_test/test_runtime_filters.py
+++ b/tests/query_test/test_runtime_filters.py
@@ -22,13 +22,10 @@ import pytest
import re
import time
-from beeswaxd.BeeswaxService import QueryState
-from tests.common.environ import build_flavor_timeout
-from tests.common.environ import ImpalaTestClusterProperties
+from tests.common.environ import build_flavor_timeout, ImpalaTestClusterProperties
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfEC
-from tests.common.skip import SkipIfLocal, SkipIfIsilon
+from tests.common.skip import SkipIfEC, SkipIfLocal, SkipIfFS
from tests.common.test_dimensions import add_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.verifiers.metric_verifier import MetricVerifier
@@ -49,7 +46,7 @@ build_runs_slowly = ImpalaTestClusterProperties.get_instance().runs_slowly()
# what tests to run for non-HDFS platforms
@pytest.mark.execute_serially
@SkipIfLocal.multiple_impalad
-@SkipIfIsilon.jira(reason="IMPALA-6998")
+@SkipIfFS.late_filters
class TestRuntimeFilters(ImpalaTestSuite):
@classmethod
def get_workload(cls):
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index ab64c6b64..389467be8 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -34,16 +34,10 @@ from testdata.common import widetable
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import (
SkipIf,
- SkipIfS3,
- SkipIfOzone,
- SkipIfGCS,
- SkipIfCOS,
- SkipIfABFS,
- SkipIfADLS,
+ SkipIfFS,
SkipIfEC,
SkipIfHive2,
SkipIfHive3,
- SkipIfIsilon,
SkipIfLocal,
SkipIfNotHdfsMinicluster)
from tests.common.test_dimensions import (
@@ -57,8 +51,7 @@ from tests.common.test_result_verifier import (
QueryTestResult,
parse_result_rows)
from tests.common.test_vector import ImpalaTestDimension
-from tests.util.filesystem_utils import IS_HDFS, WAREHOUSE, get_fs_path
-from tests.util.hdfs_util import NAMENODE
+from tests.util.filesystem_utils import IS_HDFS, get_fs_path
from tests.util.get_parquet_metadata import get_parquet_metadata
from tests.util.parse_util import get_bytes_summary_stats_counter
from tests.util.test_file_parser import QueryTestSectionReader
@@ -482,14 +475,7 @@ class TestParquet(ImpalaTestSuite):
assert len(result.data) == 1
assert "4294967294" in result.data
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
def test_multi_compression_types(self, vector, unique_database):
"""IMPALA-5448: Tests that parquet splits with multi compression types are counted
correctly. Cases tested:
@@ -616,13 +602,7 @@ class TestParquet(ImpalaTestSuite):
assert len(result.data) == 1
assert "AAAAAAAACPKFFAAA" in result.data
- @SkipIfS3.hdfs_block_size
- @SkipIfOzone.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
- @SkipIfIsilon.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfLocal.multiple_impalad
@SkipIfEC.fix_later
def test_misaligned_parquet_row_groups(self, vector):
@@ -676,13 +656,7 @@ class TestParquet(ImpalaTestSuite):
total += int(n)
assert total == num_scanners_with_no_reads
- @SkipIfS3.hdfs_block_size
- @SkipIfOzone.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
- @SkipIfIsilon.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfLocal.multiple_impalad
@SkipIfEC.fix_later
def test_multiple_blocks_mt_dop(self, vector):
@@ -728,13 +702,7 @@ class TestParquet(ImpalaTestSuite):
finally:
self.client.clear_configuration()
- @SkipIfS3.hdfs_block_size
- @SkipIfOzone.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
- @SkipIfIsilon.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfLocal.multiple_impalad
@SkipIfEC.fix_later
def test_multiple_blocks(self, vector):
@@ -747,13 +715,7 @@ class TestParquet(ImpalaTestSuite):
# there are 6 blocks and 3 scan nodes.
self._multiple_blocks_helper(table_name, 40000, ranges_per_node=2)
- @SkipIfS3.hdfs_block_size
- @SkipIfOzone.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
- @SkipIfIsilon.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfLocal.multiple_impalad
@SkipIfEC.fix_later
def test_multiple_blocks_one_row_group(self, vector):
@@ -1410,15 +1372,9 @@ class TestTextScanRangeLengths(ImpalaTestSuite):
del new_vector.get_value('exec_option')['abort_on_error']
self.run_test_case('QueryTest/dateless_timestamp_text', new_vector, unique_database)
+
# Missing Coverage: No coverage for truncated files errors or scans.
-@SkipIfS3.hive
-@SkipIfOzone.hive
-@SkipIfGCS.hive
-@SkipIfCOS.hive
-@SkipIfABFS.hive
-@SkipIfADLS.hive
-@SkipIfIsilon.hive
-@SkipIfLocal.hive
+@SkipIfFS.hive
class TestScanTruncatedFiles(ImpalaTestSuite):
@classmethod
def get_workload(cls):
@@ -1498,14 +1454,8 @@ class TestOrc(ImpalaTestSuite):
lambda v: v.get_value('table_format').file_format == 'orc')
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('orc_schema_resolution', 0, 1))
- @SkipIfS3.hdfs_block_size
- @SkipIfOzone.hdfs_block_size
- @SkipIfGCS.hdfs_block_size
- @SkipIfCOS.hdfs_block_size
- @SkipIfABFS.hdfs_block_size
- @SkipIfADLS.hdfs_block_size
+ @SkipIfFS.hdfs_block_size
@SkipIfEC.fix_later
- @SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
def test_misaligned_orc_stripes(self, vector, unique_database):
self._build_lineitem_table_helper(unique_database, 'lineitem_threeblocks',
@@ -1577,14 +1527,7 @@ class TestOrc(ImpalaTestSuite):
# queries that hang in some cases (IMPALA-9345). It would be possible to separate
# the tests that use Hive and run most tests on S3, but I think that running these on
# S3 doesn't add too much coverage.
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@SkipIfHive3.non_acid
def test_type_conversions_hive2(self, vector, unique_database):
# Create "illtypes" tables whose columns can't match the underlining ORC file's.
@@ -1629,14 +1572,7 @@ class TestOrc(ImpalaTestSuite):
# queries that hang in some cases (IMPALA-9345). It would be possible to separate
# the tests that use Hive and run most tests on S3, but I think that running these on
# S3 doesn't add too much coverage.
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@SkipIfHive2.acid
def test_type_conversions_hive3(self, vector, unique_database):
# Create "illtypes" tables whose columns can't match the underlining ORC file's.
@@ -1756,14 +1692,7 @@ class TestOrc(ImpalaTestSuite):
self.run_test_case('QueryTest/hive2-pre-gregorian-date-orc', vector, unique_database)
- @SkipIfABFS.hive
- @SkipIfADLS.hive
- @SkipIfIsilon.hive
- @SkipIfLocal.hive
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
def test_missing_field_orc(self, unique_database):
# Test scanning orc files with missing fields in file meta.
orc_tbl_name = unique_database + ".missing_field_orc"
diff --git a/tests/stress/test_acid_stress.py b/tests/stress/test_acid_stress.py
index 4a90ab25b..96a61854d 100644
--- a/tests/stress/test_acid_stress.py
+++ b/tests/stress/test_acid_stress.py
@@ -23,8 +23,7 @@ from multiprocessing import Value
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import (SkipIf, SkipIfHive2, SkipIfS3, SkipIfGCS, SkipIfCOS,
- SkipIfDockerizedCluster, SkipIfOzone)
+from tests.common.skip import SkipIf, SkipIfHive2, SkipIfFS, SkipIfDockerizedCluster
from tests.stress.stress_util import Task, run_tasks
NUM_OVERWRITES = 2
@@ -160,10 +159,7 @@ class TestAcidInsertsBasic(TestAcidStress):
sleep_seconds=0.1)])
@SkipIfHive2.acid
- @SkipIfS3.hive
- @SkipIfOzone.hive
- @SkipIfGCS.hive
- @SkipIfCOS.hive
+ @SkipIfFS.hive
@pytest.mark.execute_serially
@pytest.mark.stress
def test_read_hive_inserts(self, unique_database):
@@ -280,8 +276,7 @@ class TestConcurrentAcidInserts(TestAcidStress):
finally:
impalad_client.close()
- @SkipIfGCS.jira(reason="IMPALA-10563")
- @SkipIfCOS.jira(reason="IMPALA-10773")
+ @SkipIfFS.stress_insert_timeouts
@SkipIfHive2.acid
@SkipIfDockerizedCluster.jira(reason="IMPALA-11189")
@pytest.mark.execute_serially
@@ -378,8 +373,7 @@ class TestFailingAcidInserts(TestAcidStress):
for i in xrange(0, num_checkers)]
run_tasks(writers + checkers)
- @SkipIfGCS.jira(reason="IMPALA-10563")
- @SkipIfCOS.jira(reason="IMPALA-10773")
+ @SkipIfFS.stress_insert_timeouts
@SkipIfDockerizedCluster.jira(reason="IMPALA-11191")
@SkipIfHive2.acid
@pytest.mark.execute_serially
diff --git a/tests/stress/test_ddl_stress.py b/tests/stress/test_ddl_stress.py
index 3cd433d56..16d71a6fe 100644
--- a/tests/stress/test_ddl_stress.py
+++ b/tests/stress/test_ddl_stress.py
@@ -18,8 +18,7 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfOzone)
+from tests.common.skip import SkipIfFS
# Number of tables to create per thread
NUM_TBLS_PER_THREAD = 10
@@ -48,14 +47,7 @@ class TestDdlStress(ImpalaTestSuite):
lambda v: (v.get_value('table_format').file_format == 'text' and
v.get_value('table_format').compression_codec == 'none'))
- @SkipIfS3.caching
- @SkipIfOzone.caching
- @SkipIfGCS.caching
- @SkipIfCOS.caching
- @SkipIfABFS.caching
- @SkipIfADLS.caching
- @SkipIfIsilon.caching
- @SkipIfLocal.caching
+ @SkipIfFS.hdfs_caching
@pytest.mark.stress
@pytest.mark.parametrize('test_index', TEST_INDICES)
def test_create_cache_many_tables(self, vector, testid_checksum, test_index):