You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2021/12/08 16:47:17 UTC
[impala] branch master updated (6e2134e -> 157086c)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 6e2134e IMPALA-10886: Fix loss of createEventId for INSERT created partitions
new 32c224f IMPALA-11042: Escape special characters in LDAP filters
new 157086c IMPALA-10771: Add Tencent COS support
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/hdfs-table-sink.cc | 1 +
be/src/runtime/io/disk-io-mgr-test.cc | 4 +-
be/src/runtime/io/disk-io-mgr.cc | 8 ++++
be/src/runtime/io/disk-io-mgr.h | 4 ++
be/src/util/hdfs-util.cc | 5 +++
be/src/util/hdfs-util.h | 4 ++
be/src/util/ldap-search-bind.cc | 43 +++++++++++++++++++---
be/src/util/ldap-search-bind.h | 9 +++++
bin/impala-config.sh | 26 +++++++++++++
.../java/org/apache/impala/catalog/HdfsTable.java | 11 +++++-
.../org/apache/impala/common/FileSystemUtil.java | 15 +++++++-
.../LdapSearchBindImpalaShellTest.java | 17 +++++++++
.../java/org/apache/impala/testutil/LdapUtil.java | 3 ++
fe/src/test/resources/users.ldif | 11 ++++++
java/executor-deps/pom.xml | 6 +++
java/pom.xml | 1 +
testdata/bin/create-load-data.sh | 9 +++--
testdata/bin/run-all.sh | 2 +-
.../common/etc/hadoop/conf/core-site.xml.py | 10 +++++
tests/authorization/test_ranger.py | 3 +-
tests/common/impala_test_suite.py | 11 ++++--
tests/common/skip.py | 23 ++++++++++++
tests/custom_cluster/test_admission_controller.py | 2 +
tests/custom_cluster/test_coordinators.py | 3 +-
tests/custom_cluster/test_hdfs_fd_caching.py | 9 +++--
.../test_hive_parquet_codec_interop.py | 3 +-
.../custom_cluster/test_hive_text_codec_interop.py | 3 +-
tests/custom_cluster/test_insert_behaviour.py | 4 +-
tests/custom_cluster/test_lineage.py | 3 +-
tests/custom_cluster/test_local_catalog.py | 3 +-
tests/custom_cluster/test_local_tz_conversion.py | 3 +-
tests/custom_cluster/test_metadata_replicas.py | 2 +
tests/custom_cluster/test_metastore_service.py | 6 +--
.../custom_cluster/test_parquet_max_page_header.py | 4 +-
tests/custom_cluster/test_permanent_udfs.py | 6 ++-
tests/custom_cluster/test_query_retries.py | 4 +-
tests/custom_cluster/test_restart_services.py | 3 +-
.../custom_cluster/test_topic_update_frequency.py | 3 +-
tests/data_errors/test_data_errors.py | 5 ++-
tests/failure/test_failpoints.py | 3 +-
tests/metadata/test_catalogd_debug_actions.py | 3 +-
tests/metadata/test_compute_stats.py | 5 ++-
tests/metadata/test_ddl.py | 4 +-
tests/metadata/test_hdfs_encryption.py | 3 +-
tests/metadata/test_hdfs_permissions.py | 3 +-
tests/metadata/test_hms_integration.py | 5 ++-
tests/metadata/test_metadata_query_statements.py | 4 +-
tests/metadata/test_partition_metadata.py | 3 +-
tests/metadata/test_refresh_partition.py | 3 +-
tests/metadata/test_views_compatibility.py | 3 +-
tests/query_test/test_acid.py | 16 +++++++-
tests/query_test/test_date_queries.py | 4 +-
tests/query_test/test_hbase_queries.py | 2 +
tests/query_test/test_hdfs_caching.py | 8 +++-
tests/query_test/test_insert_behaviour.py | 10 ++++-
tests/query_test/test_insert_parquet.py | 4 +-
tests/query_test/test_join_queries.py | 2 +
tests/query_test/test_nested_types.py | 4 ++
tests/query_test/test_observability.py | 5 ++-
tests/query_test/test_partitioning.py | 3 +-
tests/query_test/test_resource_limits.py | 3 +-
tests/query_test/test_scanners.py | 10 +++++
tests/stress/test_acid_stress.py | 5 ++-
tests/stress/test_ddl_stress.py | 3 +-
tests/util/filesystem_utils.py | 3 +-
65 files changed, 353 insertions(+), 60 deletions(-)
[impala] 01/02: IMPALA-11042: Escape special characters in LDAP filters
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 32c224f03d6909f31fd9cb58e31734f74899c15b
Author: Tamas Mate <tm...@cloudera.com>
AuthorDate: Fri Dec 3 14:00:19 2021 +0100
IMPALA-11042: Escape special characters in LDAP filters
The LDAP filter specification assigns special meaning to the following
characters: '*', '(', ')', '\', 'NUL'. These 5 characters should be
represented with their ASCII characters in LDAP filters. (RFC 4516)
Impala allows '{0}' and '{1}' notations, these patterns will be replaced
dynamically during runtime, the '{0}' is replaced with the login
username, while '{1}' is replaced with the DN of the login user. This DN
is obtained through an LDAP Search.
This commit handles the escaping of the dynamically obtained properties
(username/DN) in LDAP filters according to RFC-4516. Impala should be
configured with a properly escaped filters apart from these properties.
Example DN in directory service:
cn=Ldap\, (Test6*),ou=Users2,dc=myorg,dc=com
Example DN used in filter:
cn=Ldap\5C, \28Test6\2A\29,ou=Users2,dc=myorg,dc=com
Testing:
- Added a custom cluster unti test
- Tested the behaviour manually with OpenLDAP and Active Directory
Change-Id: Ic82e8fc0ff8bbe687632db177385f467dd200dbb
Reviewed-on: http://gerrit.cloudera.org:8080/18068
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/ldap-search-bind.cc | 43 +++++++++++++++++++---
be/src/util/ldap-search-bind.h | 9 +++++
.../LdapSearchBindImpalaShellTest.java | 17 +++++++++
.../java/org/apache/impala/testutil/LdapUtil.java | 3 ++
fe/src/test/resources/users.ldif | 11 ++++++
5 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/be/src/util/ldap-search-bind.cc b/be/src/util/ldap-search-bind.cc
index c6280f2..9c47965 100644
--- a/be/src/util/ldap-search-bind.cc
+++ b/be/src/util/ldap-search-bind.cc
@@ -92,9 +92,10 @@ bool LdapSearchBind::LdapCheckPass(const char* user, const char* pass, unsigned
if (!success) return false;
VLOG(2) << "LDAP bind successful";
- // Replace the USER_NAME_PATTERN in the filter with the username
+ // Escape special characters and replace the USER_NAME_PATTERN in the filter.
string filter = string(user_filter_);
- replace_all(filter, USER_NAME_PATTERN, user);
+ string escaped_user = EscapeFilterProperty(user);
+ replace_all(filter, USER_NAME_PATTERN, escaped_user);
// Execute the LDAP search and try to retrieve the user dn
VLOG(1) << "Trying LDAP user search for: " << user;
@@ -137,10 +138,11 @@ bool LdapSearchBind::LdapCheckFilters(string username) {
// USER_DN_PATTERN requires to determine the user dn and therefore an additional LDAP
// search.
string group_filter = group_filter_;
- replace_all(group_filter, USER_NAME_PATTERN, username);
+ string escaped_username = EscapeFilterProperty(username);
+ replace_all(group_filter, USER_NAME_PATTERN, escaped_username);
if (group_filter.find(USER_DN_PATTERN) != string::npos) {
string user_filter = user_filter_;
- replace_all(user_filter, USER_NAME_PATTERN, username);
+ replace_all(user_filter, USER_NAME_PATTERN, escaped_username);
vector<string> user_dns =
LdapSearchObject(ld, FLAGS_ldap_user_search_basedn.c_str(), user_filter.c_str());
if (user_dns.size() != 1) {
@@ -151,7 +153,9 @@ bool LdapSearchBind::LdapCheckFilters(string username) {
ldap_unbind_ext(ld, nullptr, nullptr);
return false;
}
- replace_all(group_filter, USER_DN_PATTERN, user_dns[0]);
+ // Escape the characters in the the DN then replace the USER_DN_PATTERN.
+ string escaped_user_dn = EscapeFilterProperty(user_dns[0]);
+ replace_all(group_filter, USER_DN_PATTERN, escaped_user_dn);
}
// Execute LDAP search for the group
@@ -165,4 +169,33 @@ bool LdapSearchBind::LdapCheckFilters(string username) {
return true;
}
+string LdapSearchBind::EscapeFilterProperty(string property) {
+ string escaped;
+ escaped.reserve(property.size() * 2);
+ for (const char& propChar : property) {
+ switch (propChar) {
+ case '\x2A': // '*'
+ escaped.append("\\2A");
+ break;
+ case '\x28': // '('
+ escaped.append("\\28");
+ break;
+ case '\x29': // ')'
+ escaped.append("\\29");
+ break;
+ case '\x5C': // '\'
+ escaped.append("\\5C");
+ break;
+ case '\x00': // 'NUL'
+ escaped.append("\\00");
+ break;
+ default:
+ escaped.push_back(propChar);
+ break;
+ }
+ }
+ escaped.shrink_to_fit();
+ return escaped;
+}
+
} // namespace impala
diff --git a/be/src/util/ldap-search-bind.h b/be/src/util/ldap-search-bind.h
index 060c26b..9bd7b1f 100644
--- a/be/src/util/ldap-search-bind.h
+++ b/be/src/util/ldap-search-bind.h
@@ -57,6 +57,15 @@ class LdapSearchBind : public ImpalaLdap {
/// which will be replace with the user dn. '{1}' requires an extra ldap search for
/// the user dn. If empty, the group check will be skipped.
std::string group_filter_;
+
+ /// The LDAP filter specification assigns special meaning to the following characters:
+ /// '*', '(', ')', '\', 'NUL'. These 5 characters should be escaped with the backslash
+ /// escape character, followed by the two character ASCII hexadecimal representation of
+ /// the character.
+ /// This function should only be called on internal properties, currently these are
+ /// the results of '{0}' and '{1}' resolution. Impala should be configured with properly
+ /// escaped filters apart from '{0}' and '{1}'.
+ std::string EscapeFilterProperty(string property);
};
} // namespace impala
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapSearchBindImpalaShellTest.java b/fe/src/test/java/org/apache/impala/customcluster/LdapSearchBindImpalaShellTest.java
index 01520f1..b92a8f9 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapSearchBindImpalaShellTest.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapSearchBindImpalaShellTest.java
@@ -136,4 +136,21 @@ public class LdapSearchBindImpalaShellTest extends LdapImpalaShellTest {
command = buildCommand(query, "hs2-http", "invalid", "123", "/cliservice");
RunShellCommand.Run(command, /* shouldSucceed */ false, "", "");
}
+
+ /**
+ * Test group search filter validity when there is an escaped character in the user DN.
+ */
+ @Test
+ public void testEscapedCharactersInDN() throws Exception {
+ setUp("--ldap_user_search_basedn=dc=myorg,dc=com "
+ + "--ldap_group_search_basedn=ou=Groups,dc=myorg,dc=com "
+ + "--ldap_user_filter=(cn={0}) "
+ + "--ldap_group_filter=(uniqueMember={1}) ");
+ String query = "select logged_in_user()";
+
+ // Authentications should succeed with user who has escaped character in its DN
+ String[] command =
+ buildCommand(query, "hs2-http", TEST_USER_6, TEST_PASSWORD_6, "/cliservice");
+ RunShellCommand.Run(command, /* shouldSucceed */ true, TEST_USER_6, "");
+ }
}
diff --git a/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java b/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java
index fb10e07..625abb1 100644
--- a/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java
+++ b/fe/src/test/java/org/apache/impala/testutil/LdapUtil.java
@@ -35,6 +35,9 @@ public class LdapUtil {
public static final String TEST_USER_5 = "Test5Ldap";
public static final String TEST_PASSWORD_5 = "klmnop";
+ public static final String TEST_USER_6 = "Ldap\\, (Test6*)";
+ public static final String TEST_PASSWORD_6 = "qrstu";
+
// TEST_USER_1 and TEST_USER_2 are members of this group.
public static final String TEST_USER_GROUP = "group1";
diff --git a/fe/src/test/resources/users.ldif b/fe/src/test/resources/users.ldif
index b452cdb..d5e85be 100644
--- a/fe/src/test/resources/users.ldif
+++ b/fe/src/test/resources/users.ldif
@@ -69,12 +69,23 @@ sn: Ldap
uid: ldaptest5
userPassword: klmnop
+dn: cn=Ldap\, (Test6*),ou=Users2,dc=myorg,dc=com
+objectClass: inetOrgPerson
+objectClass: organizationalPerson
+objectClass: person
+objectClass: top
+cn: Ldap\, (Test6*)
+sn: Ldap
+uid: ldaptes6
+userPassword: qrstu
+
dn: cn=group1,ou=Groups,dc=myorg,dc=com
objectClass: top
objectClass: groupOfUniqueNames
cn: group1
uniqueMember: cn=Test1Ldap,ou=Users,dc=myorg,dc=com
uniqueMember: cn=Test2Ldap,ou=Users,dc=myorg,dc=com
+uniqueMember: cn=Ldap\, (Test6*),ou=Users2,dc=myorg,dc=com
dn: cn=group2,ou=Groups,dc=myorg,dc=com
objectClass: top
[impala] 02/02: IMPALA-10771: Add Tencent COS support
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 157086cb80cb54c6565da25cc327621f6b2b8513
Author: Fucun Chu <ch...@hotmail.com>
AuthorDate: Tue May 18 13:11:31 2021 +0800
IMPALA-10771: Add Tencent COS support
This patch adds support for COS(Cloud Object Storage). Using the
hadoop-cos, the implementation is similar to other remote FileSystems.
New flags for COS:
- num_cos_io_threads: Number of COS I/O threads. Defaults to be 16.
Follow-up:
- Support for caching COS file handles will be addressed in
IMPALA-10772.
- test_concurrent_inserts and test_failing_inserts in
test_acid_stress.py are skipped due to slow file listing on
COS (IMPALA-10773).
Tests:
- Upload hdfs test data to a COS bucket. Modify all locations in HMS
DB to point to the COS bucket. Remove some hdfs caching params.
Run CORE tests.
Change-Id: Idce135a7591d1b4c74425e365525be3086a39821
Reviewed-on: http://gerrit.cloudera.org:8080/17503
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/hdfs-table-sink.cc | 1 +
be/src/runtime/io/disk-io-mgr-test.cc | 4 +++-
be/src/runtime/io/disk-io-mgr.cc | 8 +++++++
be/src/runtime/io/disk-io-mgr.h | 4 ++++
be/src/util/hdfs-util.cc | 5 +++++
be/src/util/hdfs-util.h | 4 ++++
bin/impala-config.sh | 26 ++++++++++++++++++++++
.../java/org/apache/impala/catalog/HdfsTable.java | 11 ++++++++-
.../org/apache/impala/common/FileSystemUtil.java | 15 ++++++++++++-
java/executor-deps/pom.xml | 6 +++++
java/pom.xml | 1 +
testdata/bin/create-load-data.sh | 9 ++++----
testdata/bin/run-all.sh | 2 +-
.../common/etc/hadoop/conf/core-site.xml.py | 10 +++++++++
tests/authorization/test_ranger.py | 3 ++-
tests/common/impala_test_suite.py | 11 ++++++---
tests/common/skip.py | 23 +++++++++++++++++++
tests/custom_cluster/test_admission_controller.py | 2 ++
tests/custom_cluster/test_coordinators.py | 3 ++-
tests/custom_cluster/test_hdfs_fd_caching.py | 9 ++++----
.../test_hive_parquet_codec_interop.py | 3 ++-
.../custom_cluster/test_hive_text_codec_interop.py | 3 ++-
tests/custom_cluster/test_insert_behaviour.py | 4 +++-
tests/custom_cluster/test_lineage.py | 3 ++-
tests/custom_cluster/test_local_catalog.py | 3 ++-
tests/custom_cluster/test_local_tz_conversion.py | 3 ++-
tests/custom_cluster/test_metadata_replicas.py | 2 ++
tests/custom_cluster/test_metastore_service.py | 6 ++---
.../custom_cluster/test_parquet_max_page_header.py | 4 +++-
tests/custom_cluster/test_permanent_udfs.py | 6 ++++-
tests/custom_cluster/test_query_retries.py | 4 +++-
tests/custom_cluster/test_restart_services.py | 3 ++-
.../custom_cluster/test_topic_update_frequency.py | 3 ++-
tests/data_errors/test_data_errors.py | 5 ++++-
tests/failure/test_failpoints.py | 3 ++-
tests/metadata/test_catalogd_debug_actions.py | 3 ++-
tests/metadata/test_compute_stats.py | 5 ++++-
tests/metadata/test_ddl.py | 4 +++-
tests/metadata/test_hdfs_encryption.py | 3 ++-
tests/metadata/test_hdfs_permissions.py | 3 ++-
tests/metadata/test_hms_integration.py | 5 ++++-
tests/metadata/test_metadata_query_statements.py | 4 +++-
tests/metadata/test_partition_metadata.py | 3 ++-
tests/metadata/test_refresh_partition.py | 3 ++-
tests/metadata/test_views_compatibility.py | 3 ++-
tests/query_test/test_acid.py | 16 ++++++++++++-
tests/query_test/test_date_queries.py | 4 +++-
tests/query_test/test_hbase_queries.py | 2 ++
tests/query_test/test_hdfs_caching.py | 8 +++++--
tests/query_test/test_insert_behaviour.py | 10 ++++++++-
tests/query_test/test_insert_parquet.py | 4 +++-
tests/query_test/test_join_queries.py | 2 ++
tests/query_test/test_nested_types.py | 4 ++++
tests/query_test/test_observability.py | 5 ++++-
tests/query_test/test_partitioning.py | 3 ++-
tests/query_test/test_resource_limits.py | 3 ++-
tests/query_test/test_scanners.py | 10 +++++++++
tests/stress/test_acid_stress.py | 5 ++++-
tests/stress/test_ddl_stress.py | 3 ++-
tests/util/filesystem_utils.py | 3 ++-
60 files changed, 275 insertions(+), 55 deletions(-)
diff --git a/be/src/exec/hdfs-table-sink.cc b/be/src/exec/hdfs-table-sink.cc
index c1b0e87..b65e338 100644
--- a/be/src/exec/hdfs-table-sink.cc
+++ b/be/src/exec/hdfs-table-sink.cc
@@ -460,6 +460,7 @@ Status HdfsTableSink::CreateNewTmpFile(RuntimeState* state,
IsABFSPath(tmp_hdfs_file_name_cstr) ||
IsADLSPath(tmp_hdfs_file_name_cstr) ||
IsGcsPath(tmp_hdfs_file_name_cstr) ||
+ IsCosPath(tmp_hdfs_file_name_cstr) ||
IsOzonePath(tmp_hdfs_file_name_cstr)) {
// On S3A, the file cannot be stat'ed until after it's closed, and even so, the block
// size reported will be just the filesystem default. Similarly, the block size
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc
index d9a0b40..f491f59 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -53,6 +53,7 @@ DECLARE_int32(num_s3_io_threads);
DECLARE_int32(num_adls_io_threads);
DECLARE_int32(num_abfs_io_threads);
DECLARE_int32(num_gcs_io_threads);
+DECLARE_int32(num_cos_io_threads);
DECLARE_int32(num_ozone_io_threads);
DECLARE_int32(num_oss_io_threads);
DECLARE_int32(num_remote_hdfs_file_oper_io_threads);
@@ -1717,7 +1718,8 @@ TEST_F(DiskIoMgrTest, VerifyNumThreadsParameter) {
+ FLAGS_num_s3_io_threads + FLAGS_num_adls_io_threads + FLAGS_num_abfs_io_threads
+ FLAGS_num_ozone_io_threads + FLAGS_num_oss_io_threads
+ FLAGS_num_remote_hdfs_file_oper_io_threads
- + FLAGS_num_s3_file_oper_io_threads + FLAGS_num_gcs_io_threads;
+ + FLAGS_num_s3_file_oper_io_threads + FLAGS_num_gcs_io_threads
+ + FLAGS_num_cos_io_threads;
// Verify num_io_threads_per_rotational_disk and num_io_threads_per_solid_state_disk.
// Since we do not have control over which disk is used, we check for either type
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 8676811..aef83f9 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -136,6 +136,9 @@ DEFINE_int32(num_adls_io_threads, 16, "Number of ADLS I/O threads");
// The maximum number of GCS I/O threads. TODO: choose the default empirically.
DEFINE_int32(num_gcs_io_threads, 16, "Number of GCS I/O threads");
+// The maximum number of GCS I/O threads. TODO: choose the default empirically.
+DEFINE_int32(num_cos_io_threads, 16, "Number of COS I/O threads");
+
// The maximum number of Ozone I/O threads. TODO: choose the default empirically.
DEFINE_int32(num_ozone_io_threads, 16, "Number of Ozone I/O threads");
@@ -477,6 +480,9 @@ Status DiskIoMgr::Init() {
} else if (i == RemoteGcsDiskId()) {
num_threads_per_disk = FLAGS_num_gcs_io_threads;
device_name = "GCS remote";
+ } else if (i == RemoteCosDiskId()) {
+ num_threads_per_disk = FLAGS_num_cos_io_threads;
+ device_name = "COS remote";
} else if (i == RemoteOzoneDiskId()) {
num_threads_per_disk = FLAGS_num_ozone_io_threads;
device_name = "Ozone remote";
@@ -817,6 +823,7 @@ int DiskIoMgr::AssignQueue(
if (IsADLSPath(file, check_default_fs)) return RemoteAdlsDiskId();
if (IsOSSPath(file, check_default_fs)) return RemoteOSSDiskId();
if (IsGcsPath(file, check_default_fs)) return RemoteGcsDiskId();
+ if (IsCosPath(file, check_default_fs)) return RemoteCosDiskId();
if (IsOzonePath(file, check_default_fs)) return RemoteOzoneDiskId();
}
// Assign to a local disk queue.
@@ -825,6 +832,7 @@ int DiskIoMgr::AssignQueue(
DCHECK(!IsADLSPath(file, check_default_fs)); // ADLS is always remote.
DCHECK(!IsOSSPath(file, check_default_fs)); // OSS/JindoFS is always remote.
DCHECK(!IsGcsPath(file, check_default_fs)); // GCS is always remote.
+ DCHECK(!IsCosPath(file, check_default_fs)); // COS is always remote.
DCHECK(!IsOzonePath(file, check_default_fs)); // Ozone is always remote.
if (disk_id == -1) {
// disk id is unknown, assign it an arbitrary one.
diff --git a/be/src/runtime/io/disk-io-mgr.h b/be/src/runtime/io/disk-io-mgr.h
index 4745a7c..9495673 100644
--- a/be/src/runtime/io/disk-io-mgr.h
+++ b/be/src/runtime/io/disk-io-mgr.h
@@ -333,6 +333,9 @@ class DiskIoMgr : public CacheLineAligned {
/// The disk ID (and therefore disk_queues_ index) used for GCS accesses.
int RemoteGcsDiskId() const { return num_local_disks() + REMOTE_GCS_DISK_OFFSET; }
+ /// The disk ID (and therefore disk_queues_ index) used for COS accesses.
+ int RemoteCosDiskId() const { return num_local_disks() + REMOTE_COS_DISK_OFFSET; }
+
/// The disk ID (and therefore disk_queues_ index) used for Ozone accesses.
int RemoteOzoneDiskId() const { return num_local_disks() + REMOTE_OZONE_DISK_OFFSET; }
@@ -390,6 +393,7 @@ class DiskIoMgr : public CacheLineAligned {
REMOTE_ADLS_DISK_OFFSET,
REMOTE_ABFS_DISK_OFFSET,
REMOTE_GCS_DISK_OFFSET,
+ REMOTE_COS_DISK_OFFSET,
REMOTE_OZONE_DISK_OFFSET,
REMOTE_DFS_DISK_FILE_OPER_OFFSET,
REMOTE_S3_DISK_FILE_OPER_OFFSET,
diff --git a/be/src/util/hdfs-util.cc b/be/src/util/hdfs-util.cc
index b2d9193..456dd8a 100644
--- a/be/src/util/hdfs-util.cc
+++ b/be/src/util/hdfs-util.cc
@@ -35,6 +35,7 @@ const char* FILESYS_PREFIX_ABFS = "abfs://";
const char* FILESYS_PREFIX_ABFS_SEC = "abfss://";
const char* FILESYS_PREFIX_ADL = "adl://";
const char* FILESYS_PREFIX_GCS = "gs://";
+const char* FILESYS_PREFIX_COS = "cosn://";
const char* FILESYS_PREFIX_OZONE = "o3fs://";
const char* FILESYS_PREFIX_OFS = "ofs://";
const char* FILESYS_PREFIX_OSS = "oss://";
@@ -123,6 +124,10 @@ bool IsGcsPath(const char* path, bool check_default_fs) {
// o3fs and ofs uses the same transport implementation, so they should share
// the same thread pool.
+bool IsCosPath(const char* path, bool check_default_fs) {
+ return IsSpecificPath(path, FILESYS_PREFIX_COS, check_default_fs);
+}
+
bool IsOzonePath(const char* path, bool check_default_fs) {
return IsSpecificPath(path, FILESYS_PREFIX_OZONE, check_default_fs)
|| IsSpecificPath(path, FILESYS_PREFIX_OFS, check_default_fs);
diff --git a/be/src/util/hdfs-util.h b/be/src/util/hdfs-util.h
index f150299..8de3fcc 100644
--- a/be/src/util/hdfs-util.h
+++ b/be/src/util/hdfs-util.h
@@ -31,6 +31,7 @@ extern const char* FILESYS_PREFIX_ABFS;
extern const char* FILESYS_PREFIX_ABFS_SEC;
extern const char* FILESYS_PREFIX_ADL;
extern const char* FILESYS_PREFIX_GCS;
+extern const char* FILESYS_PREFIX_COS;
extern const char* FILESYS_PREFIX_OZONE;
/// Utility function to get error messages from HDFS. This function takes prefix/file and
@@ -73,6 +74,9 @@ bool IsOSSPath(const char* path, bool check_default_fs = true);
/// Returns true iff the path refers to a location on an GCS filesystem.
bool IsGcsPath(const char* path, bool check_default_fs = true);
+/// Returns true iff the path refers to a location on an COS filesystem.
+bool IsCosPath(const char* path, bool check_default_fs = true);
+
/// Returns true iff the path refers to a location on an Ozone filesystem.
bool IsOzonePath(const char* path, bool check_default_fs = true);
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 4caeb53..a414864 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -200,6 +200,7 @@ export ARCH_NAME=$(uname -p)
export IMPALA_HUDI_VERSION=0.5.0-incubating
export IMPALA_KITE_VERSION=1.1.0
export IMPALA_ORC_JAVA_VERSION=1.6.2
+export IMPALA_COS_VERSION=3.1.0-5.9.3
# When IMPALA_(CDP_COMPONENT)_URL are overridden, they may contain '$(platform_label)'
# which will be substituted for the CDP platform label in bootstrap_toolchain.py
@@ -379,6 +380,10 @@ export GOOGLE_CLOUD_PROJECT_ID="${GOOGLE_CLOUD_PROJECT_ID-}"
export GOOGLE_CLOUD_SERVICE_ACCOUNT="${GOOGLE_CLOUD_SERVICE_ACCOUNT-}"
export GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS-}"
export GCS_BUCKET="${GCS_BUCKET-}"
+export COS_SECRET_ID="${COS_SECRET_ID-}"
+export COS_SECRET_KEY="${COS_SECRET_KEY-}"
+export COS_REGION="${COS_REGION-}"
+export COS_BUCKET="${COS_BUCKET-}"
export HDFS_REPLICATION="${HDFS_REPLICATION-3}"
export ISILON_NAMENODE="${ISILON_NAMENODE-}"
# Internal and external interfaces that test cluster services will listen on. The
@@ -542,6 +547,26 @@ elif [ "${TARGET_FILESYSTEM}" = "gs" ]; then
fi
DEFAULT_FS="gs://${GCS_BUCKET}"
export DEFAULT_FS
+elif [ "${TARGET_FILESYSTEM}" = "cosn" ]; then
+ # Basic error checking
+ if [[ "${COS_SECRET_ID}" = "" ]]; then
+ echo "COS_SECRET_ID cannot be an empty string for COS"
+ return 1
+ fi
+ if [[ "${COS_SECRET_KEY}" = "" ]]; then
+ echo "COS_SECRET_KEY cannot be an empty string for COS"
+ return 1
+ fi
+ if [[ "${COS_REGION}" = "" ]]; then
+ echo "COS_REGION cannot be an empty string for COS"
+ return 1
+ fi
+ if [[ "${COS_BUCKET}" = "" ]]; then
+ echo "COS_BUCKET cannot be an empty string for COS"
+ return 1
+ fi
+ DEFAULT_FS="cosn://${COS_BUCKET}"
+ export DEFAULT_FS
elif [ "${TARGET_FILESYSTEM}" = "isilon" ]; then
if [ "${ISILON_NAMENODE}" = "" ]; then
echo "In order to access the Isilon filesystem, ISILON_NAMENODE"
@@ -789,6 +814,7 @@ echo "IMPALA_KUDU_VERSION = $IMPALA_KUDU_VERSION"
echo "IMPALA_RANGER_VERSION = $IMPALA_RANGER_VERSION"
echo "IMPALA_ICEBERG_VERSION = $IMPALA_ICEBERG_VERSION"
echo "IMPALA_GCS_VERSION = $IMPALA_GCS_VERSION"
+echo "IMPALA_COS_VERSION = $IMPALA_COS_VERSION"
# Kerberos things. If the cluster exists and is kerberized, source
# the required environment. This is required for any hadoop tool to
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index c48d14e..35c8ce5 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -790,7 +790,7 @@ public class HdfsTable extends Table implements FeFsTable {
* permissions Impala has on the given path. If the path does not exist, recurses up
* the path until a existing parent directory is found, and inherit access permissions
* from that.
- * Always returns READ_WRITE for S3, ADLS and GCS files.
+ * Always returns READ_WRITE for S3, ADLS, GCS and COS files.
*/
private static TAccessLevel getAvailableAccessLevel(String tableName,
Path location, FsPermissionCache permCache) throws IOException {
@@ -844,6 +844,15 @@ public class HdfsTable extends Table implements FeFsTable {
// So calling getPermissions() on GCS files make no sense. Assume all GCS files have
// READ_WRITE permissions.
if (FileSystemUtil.isGCSFileSystem(fs)) return true;
+
+ // COS have different authorization models:
+ // - Directory permissions are reported as 777.
+ // - File permissions are reported as 666.
+ // - File owner is reported as the local current user.
+ // - File group is also reported as the local current user.
+ // So calling getPermissions() on COS files make no sense. Assume all COS files have
+ // READ_WRITE permissions.
+ if (FileSystemUtil.isCOSFileSystem(fs)) return true;
return false;
}
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index 7efc124..09a353a 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -68,6 +68,7 @@ public class FileSystemUtil {
public static final String SCHEME_OFS = "ofs";
public static final String SCHEME_ALLUXIO = "alluxio";
public static final String SCHEME_GCS = "gs";
+ public static final String SCHEME_COS = "cosn";
/**
* Set containing all FileSystem scheme that known to supports storage UUIDs in
@@ -95,6 +96,7 @@ public class FileSystemUtil {
.add(SCHEME_O3FS)
.add(SCHEME_OFS)
.add(SCHEME_GCS)
+ .add(SCHEME_COS)
.build();
/**
@@ -108,6 +110,7 @@ public class FileSystemUtil {
.add(SCHEME_HDFS)
.add(SCHEME_S3A)
.add(SCHEME_GCS)
+ .add(SCHEME_COS)
.build();
/**
@@ -123,6 +126,7 @@ public class FileSystemUtil {
.add(SCHEME_O3FS)
.add(SCHEME_OFS)
.add(SCHEME_GCS)
+ .add(SCHEME_COS)
.build();
/**
@@ -412,6 +416,13 @@ public class FileSystemUtil {
}
/**
+ * Returns true iff the filesystem is a CosFileSystem.
+ */
+ public static boolean isCOSFileSystem(FileSystem fs) {
+ return hasScheme(fs, SCHEME_COS);
+ }
+
+ /**
* Returns true iff the filesystem is AdlFileSystem.
*/
public static boolean isADLFileSystem(FileSystem fs) {
@@ -519,7 +530,8 @@ public class FileSystemUtil {
S3,
OZONE,
ALLUXIO,
- GCS;
+ GCS,
+ COS;
private static final Map<String, FsType> SCHEME_TO_FS_MAPPING =
ImmutableMap.<String, FsType>builder()
@@ -533,6 +545,7 @@ public class FileSystemUtil {
.put(SCHEME_OFS, OZONE)
.put(SCHEME_ALLUXIO, ALLUXIO)
.put(SCHEME_GCS, GCS)
+ .put(SCHEME_COS, COS)
.build();
/**
diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml
index 502a023..5c4b532 100644
--- a/java/executor-deps/pom.xml
+++ b/java/executor-deps/pom.xml
@@ -139,6 +139,12 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.qcloud.cos</groupId>
+ <artifactId>hadoop-cos</artifactId>
+ <version>${cos.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
diff --git a/java/pom.xml b/java/pom.xml
index 702b367..1aa677c 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -45,6 +45,7 @@ under the License.
<kite.version>${env.IMPALA_KITE_VERSION}</kite.version>
<knox.version>${env.IMPALA_KNOX_VERSION}</knox.version>
<gcs.version>${env.IMPALA_GCS_VERSION}</gcs.version>
+ <cos.version>${env.IMPALA_COS_VERSION}</cos.version>
<thrift.version>0.11.0</thrift.version>
<impala.extdatasrc.api.version>${project.version}</impala.extdatasrc.api.version>
<impala.query.event.hook.api.version>${project.version}</impala.query.event.hook.api.version>
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index 9e9f161..e452952 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -119,8 +119,8 @@ fi
TIMEOUT_PID=$!
SCHEMA_MISMATCH_ERROR="A schema change has been detected in the metadata, "
-SCHEMA_MISMATCH_ERROR+="but it cannot be loaded on Isilon, s3, gcs or local filesystem, "
-SCHEMA_MISMATCH_ERROR+="and the filesystem is ${TARGET_FILESYSTEM}".
+SCHEMA_MISMATCH_ERROR+="but it cannot be loaded on Isilon, s3, gcs, cos or local "
+SCHEMA_MISMATCH_ERROR+="filesystem, and the filesystem is ${TARGET_FILESYSTEM}".
if [[ $SKIP_METADATA_LOAD -eq 0 && "$SNAPSHOT_FILE" = "" ]]; then
run-step "Generating HBase data" create-hbase.log \
@@ -134,9 +134,10 @@ elif [ $SKIP_SNAPSHOT_LOAD -eq 0 ]; then
# Don't skip the metadata load if a schema change is detected.
if ! ${IMPALA_HOME}/testdata/bin/check-schema-diff.sh; then
if [[ "${TARGET_FILESYSTEM}" == "isilon" || "${TARGET_FILESYSTEM}" == "s3" || \
- "${TARGET_FILESYSTEM}" == "local" || "${TARGET_FILESYSTEM}" == "gs" ]] ; then
+ "${TARGET_FILESYSTEM}" == "local" || "${TARGET_FILESYSTEM}" == "gs" || \
+ "${TARGET_FILESYSTEM}" == "cosn" ]] ; then
echo "ERROR in $0 at line $LINENO: A schema change has been detected in the"
- echo "metadata, but it cannot be loaded on isilon, s3, gcs or local and the"
+ echo "metadata, but it cannot be loaded on isilon, s3, gcs, cos or local and the"
echo "target file system is ${TARGET_FILESYSTEM}. Exiting."
# Generate an explicit JUnitXML symptom report here for easier triaging
${IMPALA_HOME}/bin/generate_junitxml.py --phase=dataload \
diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh
index 247a1fa..e95476e 100755
--- a/testdata/bin/run-all.sh
+++ b/testdata/bin/run-all.sh
@@ -80,7 +80,7 @@ elif [[ ${DEFAULT_FS} == "${LOCAL_FS}" ]]; then
$IMPALA_HOME/testdata/bin/run-hive-server.sh -only_metastore 2>&1 | \
tee ${IMPALA_CLUSTER_LOGS_DIR}/run-hive-server.log
else
- # With Isilon, ABFS, ADLS or GCS we only start the Hive metastore.
+ # With Isilon, ABFS, ADLS, GCS or COS we only start the Hive metastore.
# - HDFS is not started becuase remote storage is used as the defaultFs in core-site
# - HBase is irrelevent for Impala testing with remote storage.
# - We don't yet have a good way to start YARN using a different defaultFS. Moreoever,
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py
index fd3569f..b8ba65e 100644
--- a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.py
@@ -98,6 +98,16 @@ CONFIG = {
# being too restrictive for some processes performing file-based checks, e.g.
# HiveServer2 requires permission of /tmp/hive to be at lest 733.
'fs.gs.reported.permissions': '777',
+
+ # COS configuration
+ # Note: This is needed even when not running on COS, because some frontend tests
+ # include COS paths that require initializing an COS filesystem.
+ # See ExplainTest.testScanNodeFsScheme().
+ 'fs.cosn.userinfo.secretId': '${COS_SECRET_ID}',
+ 'fs.cosn.userinfo.secretKey': '${COS_SECRET_KEY}',
+ 'fs.cosn.bucket.region': '${COS_REGION}',
+ 'fs.cosn.impl': 'org.apache.hadoop.fs.CosFileSystem',
+ 'fs.AbstractFileSystem.cosn.impl': 'org.apache.hadoop.fs.CosN',
}
if target_filesystem == 's3':
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index e4f639c..487498e 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -28,7 +28,7 @@ from subprocess import check_call
from getpass import getuser
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfLocal, SkipIfHive2, SkipIfGCS)
+ SkipIfLocal, SkipIfHive2, SkipIfGCS, SkipIfCOS)
from tests.common.test_dimensions import (create_client_protocol_dimension,
create_exec_option_dimension, create_orc_dimension)
from tests.util.hdfs_util import NAMENODE
@@ -1358,6 +1358,7 @@ class TestRanger(CustomClusterTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfHive2.ranger_auth
@CustomClusterTestSuite.with_args()
def test_hive_with_ranger_setup(self, vector):
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index df8eadc..88d6940 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -66,6 +66,7 @@ from tests.util.filesystem_utils import (
IS_ABFS,
IS_ADLS,
IS_GCS,
+ IS_COS,
IS_HDFS,
S3_BUCKET_NAME,
S3GUARD_ENABLED,
@@ -223,6 +224,7 @@ class ImpalaTestSuite(BaseTestSuite):
# ADLS: uses a mixture of azure-data-lake-store-python and the HDFS CLI (TODO:
# this should completely switch to the HDFS CLI once we test it)
# GCS: uses the HDFS CLI
+ # COS: uses the HDFS CLI
#
# 'hdfs_client' is a HDFS-specific client library, and it only works when running on
# HDFS. When using 'hdfs_client', the test must be skipped on everything other than
@@ -246,6 +248,9 @@ class ImpalaTestSuite(BaseTestSuite):
elif IS_GCS:
# GCS is implemented via HDFS command line client
cls.filesystem_client = HadoopFsCommandLineClient("GCS")
+ elif IS_COS:
+ # COS is implemented via HDFS command line client
+ cls.filesystem_client = HadoopFsCommandLineClient("COS")
# Override the shell history path so that commands run by any tests
# don't write any history into the developer's file.
@@ -1034,10 +1039,10 @@ class ImpalaTestSuite(BaseTestSuite):
tf_dimensions = ImpalaTestDimension('table_format', *table_formats)
else:
tf_dimensions = load_table_info_dimension(cls.get_workload(), exploration_strategy)
- # If 'skip_hbase' is specified or the filesystem is isilon, s3, GCS(gs) or local,
- # we don't need the hbase dimension.
+ # If 'skip_hbase' is specified or the filesystem is isilon, s3, GCS(gs), COS(cosn) or
+ # local, we don't need the hbase dimension.
if pytest.config.option.skip_hbase or TARGET_FILESYSTEM.lower() \
- in ['s3', 'isilon', 'local', 'abfs', 'adls', 'gs']:
+ in ['s3', 'isilon', 'local', 'abfs', 'adls', 'gs', 'cosn']:
for tf_dimension in tf_dimensions:
if tf_dimension.value.file_format == "hbase":
tf_dimensions.remove(tf_dimension)
diff --git a/tests/common/skip.py b/tests/common/skip.py
index dd8ef88..a9099e4 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -32,6 +32,7 @@ from tests.util.filesystem_utils import (
IS_ABFS,
IS_ADLS,
IS_GCS,
+ IS_COS,
IS_EC,
IS_HDFS,
IS_ISILON,
@@ -128,6 +129,28 @@ class SkipIfGCS:
reason="Flakiness due to unpredictable listing times on GCS.")
+class SkipIfCOS:
+
+ # These are skipped due to product limitations.
+ caching = pytest.mark.skipif(IS_COS, reason="SET CACHED not implemented for COS")
+ hive = pytest.mark.skipif(IS_COS, reason="Hive doesn't work with COS")
+ hdfs_block_size = pytest.mark.skipif(IS_COS, reason="COS uses it's own block size")
+ hdfs_acls = pytest.mark.skipif(IS_COS, reason="HDFS acls are not supported on COS")
+ jira = partial(pytest.mark.skipif, IS_COS)
+ hdfs_encryption = pytest.mark.skipif(IS_COS,
+ reason="HDFS encryption is not supported with COS")
+
+ # These need test infra work to re-enable.
+ udfs = pytest.mark.skipif(IS_COS, reason="udas/udfs not copied to COS")
+ datasrc = pytest.mark.skipif(IS_COS, reason="data sources not copied to COS")
+ hbase = pytest.mark.skipif(IS_COS, reason="HBase not started with COS")
+ qualified_path = pytest.mark.skipif(IS_COS,
+ reason="Tests rely on HDFS qualified paths, IMPALA-1872")
+ variable_listing_times = pytest.mark.skipif(IS_COS,
+ reason="Flakiness due to unpredictable listing times on COS.")
+ eventually_consistent = pytest.mark.skipif(IS_COS,
+ reason="The client is slow to realize changes to file metadata")
+
class SkipIfKudu:
no_hybrid_clock = pytest.mark.skipif(
get_kudu_master_flag("--use_hybrid_clock") == "false",
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 4445942..3330b3d 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -38,6 +38,7 @@ from tests.common.resource_pool_config import ResourcePoolConfig
from tests.common.skip import (
SkipIfS3,
SkipIfGCS,
+ SkipIfCOS,
SkipIfABFS,
SkipIfADLS,
SkipIfEC,
@@ -445,6 +446,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfEC.fix_later
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index 1be16dd..cc59fa0 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -25,7 +25,7 @@ from subprocess import check_call
from tests.util.filesystem_utils import get_fs_path
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS,
- SkipIfIsilon, SkipIfGCS, SkipIfLocal)
+ SkipIfIsilon, SkipIfGCS, SkipIfCOS, SkipIfLocal)
LOG = logging.getLogger('test_coordinators')
LOG.setLevel(level=logging.DEBUG)
@@ -321,6 +321,7 @@ class TestCoordinators(CustomClusterTestSuite):
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@SkipIfABFS.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index 1ccfd48..1623a17 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -22,7 +22,8 @@ from tests.common.skip import SkipIfLocal
from tests.util.filesystem_utils import (
IS_ISILON,
IS_ADLS,
- IS_GCS)
+ IS_GCS,
+ IS_COS)
from time import sleep
@SkipIfLocal.hdfs_fd_caching
@@ -131,7 +132,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
# Caching applies to HDFS, S3, and ABFS files. If this is HDFS, S3, or ABFS, then
# verify that caching works. Otherwise, verify that file handles are not cached.
- if IS_ADLS or IS_ISILON or IS_GCS:
+ if IS_ADLS or IS_ISILON or IS_GCS or IS_COS:
caching_expected = False
else:
caching_expected = True
@@ -147,7 +148,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
handle_timeout = 5
# Only test eviction on platforms where caching is enabled.
- if IS_ADLS or IS_ISILON or IS_GCS:
+ if IS_ADLS or IS_ISILON or IS_GCS or IS_COS:
return
caching_expected = True
self.run_fd_caching_test(vector, caching_expected, cache_capacity, handle_timeout)
@@ -175,7 +176,7 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
eviction_timeout_secs = 5
# Only test eviction on platforms where caching is enabled.
- if IS_ADLS or IS_ISILON or IS_GCS:
+ if IS_ADLS or IS_ISILON or IS_GCS or IS_COS:
return
# Maximum number of file handles cached.
diff --git a/tests/custom_cluster/test_hive_parquet_codec_interop.py b/tests/custom_cluster/test_hive_parquet_codec_interop.py
index 570f8d0..51ff2cf 100644
--- a/tests/custom_cluster/test_hive_parquet_codec_interop.py
+++ b/tests/custom_cluster/test_hive_parquet_codec_interop.py
@@ -22,7 +22,7 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.util.event_processor_utils import EventProcessorUtils
from tests.common.environ import HIVE_MAJOR_VERSION
-from tests.common.skip import SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
from tests.util.filesystem_utils import get_fs_path
@@ -54,6 +54,7 @@ class TestParquetInterop(CustomClusterTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("-convert_legacy_hive_parquet_utc_timestamps=true "
"-hdfs_zone_info_zip=%s" % get_fs_path("/test-warehouse/tzdb/2017c.zip"))
diff --git a/tests/custom_cluster/test_hive_text_codec_interop.py b/tests/custom_cluster/test_hive_text_codec_interop.py
index cf6e2ee..4b28138 100644
--- a/tests/custom_cluster/test_hive_text_codec_interop.py
+++ b/tests/custom_cluster/test_hive_text_codec_interop.py
@@ -21,7 +21,7 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import HIVE_MAJOR_VERSION
-from tests.common.skip import SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
@@ -53,6 +53,7 @@ class TestTextInterop(CustomClusterTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@pytest.mark.execute_serially
def test_hive_impala_interop(self, unique_database, cluster_properties):
"""Tests compressed text file written by Hive with different codecs
diff --git a/tests/custom_cluster/test_insert_behaviour.py b/tests/custom_cluster/test_insert_behaviour.py
index 031a54c..57c222e 100644
--- a/tests/custom_cluster/test_insert_behaviour.py
+++ b/tests/custom_cluster/test_insert_behaviour.py
@@ -18,7 +18,8 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal, SkipIfGCS
+from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal,
+ SkipIfGCS, SkipIfCOS)
from tests.util.filesystem_utils import IS_ISILON, WAREHOUSE
from tests.util.hdfs_util import (
HdfsConfig,
@@ -29,6 +30,7 @@ TEST_TBL = "insert_inherit_permission"
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+@SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index e01df0d..62b9d44 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -26,7 +26,7 @@ import tempfile
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfGCS, SkipIfCOS
LOG = logging.getLogger(__name__)
@@ -153,6 +153,7 @@ class TestLineage(CustomClusterTestSuite):
@SkipIfADLS.hbase
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--lineage_event_log_dir={0}"
.format(LINEAGE_TESTS_DIR))
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index ca6affb..63b0cbb 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -27,7 +27,7 @@ import time
from multiprocessing.pool import ThreadPool
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import (SkipIfHive2, SkipIfS3, SkipIfABFS, SkipIfGCS,
+from tests.common.skip import (SkipIfHive2, SkipIfS3, SkipIfABFS, SkipIfGCS, SkipIfCOS,
SkipIfADLS, SkipIfIsilon, SkipIfLocal)
from tests.util.filesystem_utils import WAREHOUSE
@@ -535,6 +535,7 @@ class TestFullAcid(CustomClusterTestSuite):
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
@pytest.mark.execute_serially
diff --git a/tests/custom_cluster/test_local_tz_conversion.py b/tests/custom_cluster/test_local_tz_conversion.py
index fefcfc1..241750b 100644
--- a/tests/custom_cluster/test_local_tz_conversion.py
+++ b/tests/custom_cluster/test_local_tz_conversion.py
@@ -19,7 +19,7 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_vector import ImpalaTestDimension
-from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfABFS, SkipIfADLS, SkipIfS3, SkipIfGCS, SkipIfCOS
from tests.common.test_dimensions import create_exec_option_dimension
class TestLocalTzConversion(CustomClusterTestSuite):
@@ -51,6 +51,7 @@ class TestLocalTzConversion(CustomClusterTestSuite):
@SkipIfADLS.hbase
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--use_local_tz_for_unix_timestamp_conversions=true")
def test_timestamp_functions(self, vector):
diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py
index 3e89070..3f52bcc 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -24,12 +24,14 @@ from tests.common.skip import (
SkipIfABFS,
SkipIfADLS,
SkipIfGCS,
+ SkipIfCOS,
SkipIfIsilon,
SkipIfLocal)
from tests.util.hive_utils import HiveDbWrapper
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py
index 62585c4..4569816 100644
--- a/tests/custom_cluster/test_metastore_service.py
+++ b/tests/custom_cluster/test_metastore_service.py
@@ -28,7 +28,7 @@ from hive_metastore.ttypes import SerDeInfo
from tests.util.event_processor_utils import EventProcessorUtils
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.util.filesystem_utils import (IS_S3, IS_ADLS, IS_GCS)
+from tests.util.filesystem_utils import (IS_S3, IS_ADLS, IS_GCS, IS_COS)
class TestMetastoreService(CustomClusterTestSuite):
@@ -1167,7 +1167,7 @@ class TestMetastoreService(CustomClusterTestSuite):
assert get_parts_by_names_result.dictionary is not None
# obj_dict will only be populated when the table is on HDFS
# where block locations are available.
- if not IS_S3 and not IS_GCS and not IS_ADLS:
+ if not IS_S3 and not IS_GCS and not IS_COS and not IS_ADLS:
assert len(get_parts_by_names_result.dictionary.values) > 0
else:
assert get_parts_by_names_result.dictionary is None
@@ -1195,7 +1195,7 @@ class TestMetastoreService(CustomClusterTestSuite):
assert obj_dict is not None
# obj_dict will only be populated when the table is on HDFS
# where block locations are available.
- if not IS_S3 and not IS_GCS and not IS_ADLS:
+ if not IS_S3 and not IS_GCS and not IS_COS and not IS_ADLS:
assert len(obj_dict.values) > 0
def __assert_no_filemd(self, filemetadata, obj_dict):
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index e6f80f2..9bd5ca3 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -24,7 +24,8 @@ import string
import subprocess
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfGCS
+from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfGCS,
+ SkipIfCOS)
class TestParquetMaxPageHeader(CustomClusterTestSuite):
'''This tests large page headers in parquet files. Parquet page header size can
@@ -102,6 +103,7 @@ class TestParquetMaxPageHeader(CustomClusterTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/custom_cluster/test_permanent_udfs.py b/tests/custom_cluster/test_permanent_udfs.py
index f75e86e..2031823 100644
--- a/tests/custom_cluster/test_permanent_udfs.py
+++ b/tests/custom_cluster/test_permanent_udfs.py
@@ -25,7 +25,7 @@ import subprocess
from tempfile import mkdtemp
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfGCS,
- SkipIfLocal)
+ SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.filesystem_utils import get_fs_path
@@ -164,6 +164,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@@ -187,6 +188,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@@ -253,6 +255,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@@ -317,6 +320,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py
index 04bcd89..2b3d9aa 100644
--- a/tests/custom_cluster/test_query_retries.py
+++ b/tests/custom_cluster/test_query_retries.py
@@ -34,7 +34,7 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.errors import Timeout
-from tests.common.skip import SkipIfEC, SkipIfBuildType, SkipIfGCS
+from tests.common.skip import SkipIfEC, SkipIfBuildType, SkipIfGCS, SkipIfCOS
from tests.common.skip import SkipIfNotHdfsMinicluster
# The BE krpc port of the impalad to simulate rpc or disk errors in tests.
@@ -284,6 +284,7 @@ class TestQueryRetries(CustomClusterTestSuite):
self.__validate_web_ui_state()
@SkipIfGCS.jira(reason="IMPALA-10562")
+ @SkipIfCOS.jira(reason="IMPALA-10562")
@SkipIfBuildType.not_dev_build
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
@@ -350,6 +351,7 @@ class TestQueryRetries(CustomClusterTestSuite):
self.__validate_web_ui_state()
@SkipIfGCS.jira(reason="IMPALA-10562")
+ @SkipIfCOS.jira(reason="IMPALA-10562")
@SkipIfBuildType.not_dev_build
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index beccc27..da797ec 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -35,7 +35,7 @@ from TCLIService import TCLIService
from beeswaxd.BeeswaxService import QueryState
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfGCS
+from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfGCS, SkipIfCOS
from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
LOG = logging.getLogger(__name__)
@@ -361,6 +361,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
return 'functional-query'
@SkipIfGCS.jira(reason="IMPALA-10562")
+ @SkipIfCOS.jira(reason="IMPALA-10562")
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--shutdown_grace_period_s={grace_period} \
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index 30069fe..2162735 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -15,11 +15,12 @@ import pytest
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
@SkipIfS3.variable_listing_times
@SkipIfGCS.variable_listing_times
+@SkipIfCOS.variable_listing_times
class TestTopicUpdateFrequency(CustomClusterTestSuite):
@pytest.mark.execute_serially
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index b61e66f..a507314 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -26,7 +26,7 @@ import subprocess
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfGCS,
- SkipIfLocal)
+ SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import create_exec_option_dimension
class TestDataErrors(ImpalaTestSuite):
@@ -108,6 +108,7 @@ class TestHdfsUnknownErrors(ImpalaTestSuite):
@SkipIfS3.qualified_path
@SkipIfGCS.qualified_path
+@SkipIfCOS.qualified_path
@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
class TestHdfsScanNodeErrors(TestDataErrors):
@@ -128,6 +129,7 @@ class TestHdfsScanNodeErrors(TestDataErrors):
@SkipIfS3.qualified_path
@SkipIfGCS.qualified_path
+@SkipIfCOS.qualified_path
@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
@SkipIfLocal.qualified_path
@@ -145,6 +147,7 @@ class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
@SkipIfS3.qualified_path
@SkipIfGCS.qualified_path
+@SkipIfCOS.qualified_path
@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
class TestHdfsRcFileScanNodeErrors(TestHdfsScanNodeErrors):
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index c8244f9..d885962 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -28,7 +28,7 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import (SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.verifiers.metric_verifier import MetricVerifier
@@ -64,6 +64,7 @@ QUERIES = [
@SkipIf.skip_hbase # -skip_hbase argument specified
@SkipIfS3.hbase # S3: missing coverage: failures
@SkipIfGCS.hbase
+@SkipIfCOS.hbase
@SkipIfABFS.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase # ISILON: missing coverage: failures.
diff --git a/tests/metadata/test_catalogd_debug_actions.py b/tests/metadata/test_catalogd_debug_actions.py
index 115ae2c..57d5d5d 100644
--- a/tests/metadata/test_catalogd_debug_actions.py
+++ b/tests/metadata/test_catalogd_debug_actions.py
@@ -17,11 +17,12 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIfS3, SkipIfGCS, SkipIfCOS
@SkipIfS3.variable_listing_times
@SkipIfGCS.variable_listing_times
+@SkipIfCOS.variable_listing_times
class TestDebugActions(ImpalaTestSuite):
@pytest.mark.execute_serially
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index bbb0a40..889f0b3 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -22,7 +22,7 @@ from tests.common.environ import ImpalaTestClusterProperties
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfCatalogV2)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_single_exec_option_dimension,
@@ -115,6 +115,7 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -195,6 +196,7 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -243,6 +245,7 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 34c2f76..7584ae5 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -30,7 +30,8 @@ from tests.common.file_utils import create_table_from_orc
from tests.common.impala_test_suite import LOG
from tests.common.parametrize import UniqueDatabase
from tests.common.skip import (SkipIf, SkipIfABFS, SkipIfADLS, SkipIfKudu, SkipIfLocal,
- SkipIfCatalogV2, SkipIfHive2, SkipIfS3, SkipIfGCS)
+ SkipIfCatalogV2, SkipIfHive2, SkipIfS3, SkipIfGCS,
+ SkipIfCOS)
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import (create_exec_option_dimension,
create_client_protocol_dimension)
@@ -306,6 +307,7 @@ class TestDdlStatements(TestDdlBase):
@SkipIfHive2.orc
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@UniqueDatabase.parametrize(sync_ddl=True)
def test_create_table_like_file_orc(self, vector, unique_database):
COMPLEXTYPETBL_PATH = 'test-warehouse/managed/functional_orc_def.db/' \
diff --git a/tests/metadata/test_hdfs_encryption.py b/tests/metadata/test_hdfs_encryption.py
index 470227b..bb6cba5 100644
--- a/tests/metadata/test_hdfs_encryption.py
+++ b/tests/metadata/test_hdfs_encryption.py
@@ -20,7 +20,7 @@ import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -36,6 +36,7 @@ TMP_DIR = '/%s' % (PYWEBHDFS_TMP_DIR)
@SkipIfS3.hdfs_encryption
@SkipIfGCS.hdfs_encryption
+@SkipIfCOS.hdfs_encryption
@SkipIfABFS.hdfs_encryption
@SkipIfADLS.hdfs_encryption
@SkipIfIsilon.hdfs_encryption
diff --git a/tests/metadata/test_hdfs_permissions.py b/tests/metadata/test_hdfs_permissions.py
index 3d0cccc..e505165 100644
--- a/tests/metadata/test_hdfs_permissions.py
+++ b/tests/metadata/test_hdfs_permissions.py
@@ -17,7 +17,7 @@
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal,
- SkipIfGCS, SkipIfCatalogV2)
+ SkipIfGCS, SkipIfCOS, SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -30,6 +30,7 @@ TBL_LOC = '%s/%s' % (WAREHOUSE, TEST_TBL)
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+@SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfLocal.hdfs_client
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 383154b..a4c023b 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -32,7 +32,8 @@ from subprocess import call
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfHive2, SkipIfHive3,
- SkipIfIsilon, SkipIfGCS, SkipIfLocal, SkipIfCatalogV2)
+ SkipIfIsilon, SkipIfGCS, SkipIfCOS, SkipIfLocal,
+ SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -42,6 +43,7 @@ from tests.util.hive_utils import HiveDbWrapper, HiveTableWrapper
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -104,6 +106,7 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index d4cc273..9cd3019 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -23,7 +23,7 @@ import re
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfIsilon, SkipIfS3, SkipIfABFS, SkipIfADLS,
- SkipIfGCS, SkipIfLocal, SkipIfCatalogV2)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfCatalogV2)
from tests.common.test_dimensions import ALL_NODES_ONLY
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -78,6 +78,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@@ -170,6 +171,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/metadata/test_partition_metadata.py b/tests/metadata/test_partition_metadata.py
index 3c551a6..3630147 100644
--- a/tests/metadata/test_partition_metadata.py
+++ b/tests/metadata/test_partition_metadata.py
@@ -18,7 +18,7 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import (create_single_exec_option_dimension,
create_uncompressed_text_dimension)
from tests.util.filesystem_utils import get_fs_path, WAREHOUSE, FILESYSTEM_PREFIX
@@ -91,6 +91,7 @@ class TestPartitionMetadata(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index a07cc30..3046647 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -18,12 +18,13 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.util.filesystem_utils import get_fs_path
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index 031413f..ffb2107 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -25,7 +25,7 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader
@@ -50,6 +50,7 @@ from tests.util.test_file_parser import QueryTestSectionReader
# other on non hdfs storage.
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/query_test/test_acid.py b/tests/query_test/test_acid.py
index b8ea970..74e2fbd 100644
--- a/tests/query_test/test_acid.py
+++ b/tests/query_test/test_acid.py
@@ -25,7 +25,8 @@ from hive_metastore.ttypes import CommitTxnRequest, OpenTxnRequest
from subprocess import check_call
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIf, SkipIfHive2, SkipIfCatalogV2, SkipIfS3, SkipIfABFS,
- SkipIfADLS, SkipIfIsilon, SkipIfGCS, SkipIfLocal)
+ SkipIfADLS, SkipIfIsilon, SkipIfGCS, SkipIfCOS,
+ SkipIfLocal)
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.acid_txn import AcidTxn
@@ -47,6 +48,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -65,6 +67,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -75,6 +78,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -85,6 +89,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -97,6 +102,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -107,6 +113,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -127,6 +134,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfCatalogV2.hms_event_polling_enabled()
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -141,6 +149,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -164,6 +173,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -181,6 +191,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -200,6 +211,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -215,6 +227,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -294,6 +307,7 @@ class TestAcid(ImpalaTestSuite):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/query_test/test_date_queries.py b/tests/query_test/test_date_queries.py
index 7bc9be4..f6e5d6e 100644
--- a/tests/query_test/test_date_queries.py
+++ b/tests/query_test/test_date_queries.py
@@ -20,7 +20,8 @@
import pytest
from tests.common.file_utils import create_table_and_copy_files
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal, SkipIfGCS
+from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal, SkipIfGCS,
+ SkipIfCOS)
from tests.common.test_dimensions import (create_exec_option_dimension_from_dict,
create_client_protocol_dimension, hs2_parquet_constraint)
from tests.shell.util import ImpalaShell
@@ -72,6 +73,7 @@ class TestDateQueries(ImpalaTestSuite):
@SkipIfS3.qualified_path
@SkipIfGCS.qualified_path
+ @SkipIfCOS.qualified_path
@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
@SkipIfLocal.qualified_path
diff --git a/tests/query_test/test_hbase_queries.py b/tests/query_test/test_hbase_queries.py
index d28ef4a..a209ada 100644
--- a/tests/query_test/test_hbase_queries.py
+++ b/tests/query_test/test_hbase_queries.py
@@ -23,6 +23,7 @@ from tests.common.skip import (
SkipIfIsilon,
SkipIfS3,
SkipIfGCS,
+ SkipIfCOS,
SkipIfABFS,
SkipIfADLS,
SkipIfLocal)
@@ -68,6 +69,7 @@ class TestHBaseQueries(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 0815e58..305c376 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -26,8 +26,8 @@ from tests.common.environ import build_flavor_timeout, IS_DOCKERIZED_TEST_CLUSTE
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfEC, SkipIfDockerizedCluster,
- SkipIfCatalogV2)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfEC,
+ SkipIfDockerizedCluster, SkipIfCatalogV2)
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
from tests.util.shell_util import exec_process
@@ -35,6 +35,7 @@ from tests.util.shell_util import exec_process
# End to end test that hdfs caching is working.
@SkipIfS3.caching # S3: missing coverage: verify SET CACHED gives error
@SkipIfGCS.caching
+@SkipIfCOS.caching
@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@@ -116,6 +117,7 @@ class TestHdfsCaching(ImpalaTestSuite):
# TODO: Move this to TestHdfsCaching once we make exhaustive tests run for other workloads
@SkipIfS3.caching
@SkipIfGCS.caching
+@SkipIfCOS.caching
@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@@ -127,6 +129,7 @@ class TestHdfsCachingFallbackPath(ImpalaTestSuite):
@SkipIfS3.hdfs_encryption
@SkipIfGCS.hdfs_encryption
+ @SkipIfCOS.hdfs_encryption
@SkipIfABFS.hdfs_encryption
@SkipIfADLS.hdfs_encryption
@SkipIfIsilon.hdfs_encryption
@@ -180,6 +183,7 @@ class TestHdfsCachingFallbackPath(ImpalaTestSuite):
@SkipIfS3.caching
@SkipIfGCS.caching
+@SkipIfCOS.caching
@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index 5be8c94..8a3c6c1 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -24,7 +24,7 @@ import re
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfDockerizedCluster,
+ SkipIfGCS, SkipIfCOS, SkipIfLocal, SkipIfDockerizedCluster,
SkipIfCatalogV2)
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_S3
@@ -135,6 +135,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -198,6 +199,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -252,6 +254,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -335,6 +338,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -369,6 +373,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -451,6 +456,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@@ -507,6 +513,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.execute_query_expect_success(self.client, load_dir_query)
@SkipIfADLS.eventually_consistent
+ @SkipIfCOS.eventually_consistent
@pytest.mark.execute_serially
def test_insert_select_with_empty_resultset(self):
"""Test insert/select query won't trigger partition directory or zero size data file
@@ -578,6 +585,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
@SkipIfS3.hdfs_acls
@SkipIfGCS.hdfs_acls
+ @SkipIfCOS.hdfs_acls
@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 1a050e6..bd898a8 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -29,7 +29,7 @@ from tests.common.environ import impalad_basedir
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
from tests.common.skip import (SkipIfEC, SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfABFS,
- SkipIfADLS, SkipIfGCS)
+ SkipIfADLS, SkipIfGCS, SkipIfCOS)
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
from tests.common.test_vector import ImpalaTestDimension
@@ -540,6 +540,7 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
# IMPALA-9365 describes why HS2 is not started on non-HDFS test env.
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -561,6 +562,7 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
# TODO: Should we move this to test_parquet_stats.py?
diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py
index 669aa26..6ecd5c0 100644
--- a/tests/query_test/test_join_queries.py
+++ b/tests/query_test/test_join_queries.py
@@ -27,6 +27,7 @@ from tests.common.skip import (
SkipIfLocal,
SkipIfS3,
SkipIfGCS,
+ SkipIfCOS,
SkipIfABFS,
SkipIfADLS)
from tests.common.test_vector import ImpalaTestDimension
@@ -81,6 +82,7 @@ class TestJoinQueries(ImpalaTestSuite):
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@SkipIfABFS.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 364e063..afaeb48 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -27,6 +27,7 @@ from tests.common.skip import (
SkipIfIsilon,
SkipIfS3,
SkipIfGCS,
+ SkipIfCOS,
SkipIfABFS,
SkipIfADLS,
SkipIfEC,
@@ -272,6 +273,7 @@ class TestNestedTypesNoMtDop(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@@ -331,6 +333,7 @@ class TestNestedTypesNoMtDop(ImpalaTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfHive2.acid
def test_partitioned_table_acid(self, vector, unique_database):
"""IMPALA-6370: Test that a partitioned table with nested types can be scanned."""
@@ -833,6 +836,7 @@ class TestMaxNestingDepth(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index ba01df6..79f15a1 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -20,7 +20,8 @@ from datetime import datetime
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal, SkipIfNotHdfsMinicluster)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal,
+ SkipIfNotHdfsMinicluster)
from tests.util.filesystem_utils import IS_EC
from time import sleep
from RuntimeProfile.ttypes import TRuntimeProfileFormat
@@ -93,6 +94,7 @@ class TestObservability(ImpalaTestSuite):
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@SkipIfLocal.hbase
@SkipIfIsilon.hbase
@SkipIfABFS.hbase
@@ -673,6 +675,7 @@ class TestObservability(ImpalaTestSuite):
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@SkipIfLocal.hbase
@SkipIfIsilon.hbase
@SkipIfABFS.hbase
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 4af89f0..e138559 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -21,7 +21,7 @@ import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
from tests.common.test_dimensions import create_single_exec_option_dimension
# Tests to validate HDFS partitioning.
@@ -49,6 +49,7 @@ class TestPartitioning(ImpalaTestSuite):
# filesystem.
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
diff --git a/tests/query_test/test_resource_limits.py b/tests/query_test/test_resource_limits.py
index bb9e7c6..1e78228 100644
--- a/tests/query_test/test_resource_limits.py
+++ b/tests/query_test/test_resource_limits.py
@@ -17,7 +17,7 @@
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfEC, SkipIfLocal, SkipIfS3, SkipIfABFS,
- SkipIfGCS, SkipIfADLS)
+ SkipIfGCS, SkipIfCOS, SkipIfADLS)
from tests.common.test_dimensions import create_parquet_dimension
@@ -47,6 +47,7 @@ class TestResourceLimits(ImpalaTestSuite):
@SkipIfS3.hbase
@SkipIfGCS.hbase
+ @SkipIfCOS.hbase
@SkipIfADLS.hbase
@SkipIfABFS.hbase
@SkipIfLocal.multiple_impalad
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 314654f..f2b0c4d 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -36,6 +36,7 @@ from tests.common.skip import (
SkipIf,
SkipIfS3,
SkipIfGCS,
+ SkipIfCOS,
SkipIfABFS,
SkipIfADLS,
SkipIfEC,
@@ -470,6 +471,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
def test_multi_compression_types(self, vector, unique_database):
"""IMPALA-5448: Tests that parquet splits with multi compression types are counted
correctly. Cases tested:
@@ -589,6 +591,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@@ -647,6 +650,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@@ -687,6 +691,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@@ -704,6 +709,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@@ -1366,6 +1372,7 @@ class TestTextScanRangeLengths(ImpalaTestSuite):
# Missing Coverage: No coverage for truncated files errors or scans.
@SkipIfS3.hive
@SkipIfGCS.hive
+@SkipIfCOS.hive
@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@@ -1450,6 +1457,7 @@ class TestOrc(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfGCS.hdfs_block_size
+ @SkipIfCOS.hdfs_block_size
@SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfEC.fix_later
@@ -1525,6 +1533,7 @@ class TestOrc(ImpalaTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfHive3.non_acid
def test_type_conversions_hive2(self, vector, unique_database):
# Create "illtypes" tables whose columns can't match the underlining ORC file's.
@@ -1575,6 +1584,7 @@ class TestOrc(ImpalaTestSuite):
@SkipIfLocal.hive
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@SkipIfHive2.acid
def test_type_conversions_hive3(self, vector, unique_database):
# Create "illtypes" tables whose columns can't match the underlining ORC file's.
diff --git a/tests/stress/test_acid_stress.py b/tests/stress/test_acid_stress.py
index c997e1e..f6439ff 100644
--- a/tests/stress/test_acid_stress.py
+++ b/tests/stress/test_acid_stress.py
@@ -23,7 +23,7 @@ from multiprocessing import Value
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIf, SkipIfHive2, SkipIfS3, SkipIfGCS
+from tests.common.skip import SkipIf, SkipIfHive2, SkipIfS3, SkipIfGCS, SkipIfCOS
from tests.stress.stress_util import Task, run_tasks
NUM_OVERWRITES = 2
@@ -161,6 +161,7 @@ class TestAcidInsertsBasic(TestAcidStress):
@SkipIfHive2.acid
@SkipIfS3.hive
@SkipIfGCS.hive
+ @SkipIfCOS.hive
@pytest.mark.execute_serially
@pytest.mark.stress
def test_read_hive_inserts(self, unique_database):
@@ -278,6 +279,7 @@ class TestConcurrentAcidInserts(TestAcidStress):
impalad_client.close()
@SkipIfGCS.jira(reason="IMPALA-10563")
+ @SkipIfCOS.jira(reason="IMPALA-10773")
@SkipIfHive2.acid
@pytest.mark.execute_serially
@pytest.mark.stress
@@ -374,6 +376,7 @@ class TestFailingAcidInserts(TestAcidStress):
run_tasks(writers + checkers)
@SkipIfGCS.jira(reason="IMPALA-10563")
+ @SkipIfCOS.jira(reason="IMPALA-10773")
@SkipIfHive2.acid
@pytest.mark.execute_serially
@pytest.mark.stress
diff --git a/tests/stress/test_ddl_stress.py b/tests/stress/test_ddl_stress.py
index 71f76e1..8117cfc 100644
--- a/tests/stress/test_ddl_stress.py
+++ b/tests/stress/test_ddl_stress.py
@@ -19,7 +19,7 @@ import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon,
- SkipIfGCS, SkipIfLocal)
+ SkipIfGCS, SkipIfCOS, SkipIfLocal)
# Number of tables to create per thread
NUM_TBLS_PER_THREAD = 10
@@ -50,6 +50,7 @@ class TestDdlStress(ImpalaTestSuite):
@SkipIfS3.caching
@SkipIfGCS.caching
+ @SkipIfCOS.caching
@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index 24c960d..605b332 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -32,6 +32,7 @@ IS_HDFS = FILESYSTEM == "hdfs"
IS_ADLS = FILESYSTEM == "adls"
IS_ABFS = FILESYSTEM == "abfs"
IS_GCS = FILESYSTEM == "gs"
+IS_COS = FILESYSTEM == "cosn"
IS_EC = os.getenv("ERASURE_CODING") == "true"
# This condition satisfies both the states where one can assume a default fs
# - The environment variable is set to an empty string.
@@ -57,7 +58,7 @@ ADLS_CLIENT_SECRET = os.getenv("azure_client_secret")
# A map of FILESYSTEM values to their corresponding Scan Node types
fs_to_name = {'s3': 'S3', 'hdfs': 'HDFS', 'local': 'LOCAL', 'adls': 'ADLS',
- 'abfs': 'ADLS', 'gs': 'GCS'}
+ 'abfs': 'ADLS', 'gs': 'GCS', 'cosn': 'COS'}
def get_fs_name(fs):