You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2018/10/20 06:44:10 UTC
[1/5] impala git commit: IMPALA-7729: Fix invalidate metadata hang
when there is an upper case role name
Repository: impala
Updated Branches:
refs/heads/master 8c93a4568 -> 7a022cf36
IMPALA-7729: Fix invalidate metadata hang when there is an upper case role name
Sentry stores the role names in lower case and Impala stores the role
names based on the original input role names. IMPALA-7343 introduced
a new bulk API (listAllRolesPrivileges) from Sentry that returns a map
of role name to a set of privileges. Since Impala preserves the case
sensitivity of the role names based on the original input role names,
this causes an issue when trying to retrieve a set of privileges from
a role name that is stored in Impala, especially when the role names in
Impala differ than the ones returned by listAllRolesPrivileges. This
issue will then result in privileges with mismatch role names to never
get refreshed in the Catalogd, which causes Impalad to wait indefinitely
waiting for the privileges to be updated by Catalogd. The fix is to get
a set of privileges using the role names returned by Sentry's
listAllRoles instead of using the role names stored in Impala.
Testing:
- Added a new E2E test
- Ran all E2E authorization tests
Change-Id: I5aa6f626ad3df4e9321ed18273d045517bc099c2
Reviewed-on: http://gerrit.cloudera.org:8080/11734
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/072f3ee9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/072f3ee9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/072f3ee9
Branch: refs/heads/master
Commit: 072f3ee9045d62cceb23f1f416f3052e0024cdcd
Parents: 8c93a45
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Thu Oct 18 22:48:07 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Oct 19 21:48:28 2018 +0000
----------------------------------------------------------------------
.../org/apache/impala/catalog/Principal.java | 7 +++++-
.../java/org/apache/impala/catalog/Role.java | 7 ++++++
.../java/org/apache/impala/catalog/User.java | 7 ++++++
.../org/apache/impala/util/SentryProxy.java | 25 +++++++++++++++-----
tests/authorization/test_grant_revoke.py | 22 +++++++++++++++++
5 files changed, 61 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/072f3ee9/fe/src/main/java/org/apache/impala/catalog/Principal.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Principal.java b/fe/src/main/java/org/apache/impala/catalog/Principal.java
index d048d10..9f3f6d4 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Principal.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Principal.java
@@ -39,7 +39,7 @@ public abstract class Principal extends CatalogObjectImpl {
private static AtomicInteger principalId_ = new AtomicInteger(0);
private final CatalogObjectCache<PrincipalPrivilege> principalPrivileges_ =
- new CatalogObjectCache<>();
+ new CatalogObjectCache<>(isCaseInsensitiveKeys());
protected Principal(String principalName, TPrincipalType type,
Set<String> grantGroups) {
@@ -55,6 +55,11 @@ public abstract class Principal extends CatalogObjectImpl {
}
/**
+ * Returns true if the keys in the catalog to be treated as case insensitive.
+ */
+ protected abstract boolean isCaseInsensitiveKeys();
+
+ /**
* Adds a privilege to the principal. Returns true if the privilege was added
* successfully or false if there was a newer version of the privilege already added
* to the principal.
http://git-wip-us.apache.org/repos/asf/impala/blob/072f3ee9/fe/src/main/java/org/apache/impala/catalog/Role.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Role.java b/fe/src/main/java/org/apache/impala/catalog/Role.java
index 9cee7d2..d2e841a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Role.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Role.java
@@ -36,4 +36,11 @@ public class Role extends Principal {
Preconditions.checkArgument(
thriftPrincipal.getPrincipal_type() == TPrincipalType.ROLE);
}
+
+ @Override
+ protected boolean isCaseInsensitiveKeys() {
+ // If Sentry changes the role name to be case sensitive, make sure to update
+ // this code to return false.
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/072f3ee9/fe/src/main/java/org/apache/impala/catalog/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/User.java b/fe/src/main/java/org/apache/impala/catalog/User.java
index 2845670..9bcd7fb 100644
--- a/fe/src/main/java/org/apache/impala/catalog/User.java
+++ b/fe/src/main/java/org/apache/impala/catalog/User.java
@@ -36,4 +36,11 @@ public class User extends Principal {
Preconditions.checkArgument(
thriftPrincipal.getPrincipal_type() == TPrincipalType.USER);
}
+
+ @Override
+ protected boolean isCaseInsensitiveKeys() {
+ // If Sentry changes the user name to be case sensitive, make sure to update
+ // this code to return false.
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/072f3ee9/fe/src/main/java/org/apache/impala/util/SentryProxy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryProxy.java b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
index f74e033..acad161 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryProxy.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
@@ -169,6 +169,7 @@ public class SentryProxy {
// Assume all roles should be removed. Then query the Policy Service and remove
// roles from this set that actually exist.
Set<String> rolesToRemove = catalog_.getAuthPolicy().getAllRoleNames();
+ // The keys (role names) in listAllRolesPrivileges here are always in lower case.
Map<String, Set<TSentryPrivilege>> allRolesPrivileges =
sentryPolicyService_.listAllRolesPrivileges(processUser_);
// Read the full policy, adding new/modified roles to "updatedRoles".
@@ -195,7 +196,9 @@ public class SentryProxy {
} else {
role = catalog_.addRole(sentryRole.getRoleName(), grantGroups);
}
- refreshPrivilegesInCatalog(role, allRolesPrivileges);
+ // allRolesPrivileges keys and sentryRole.getName() are used here since they both
+ // come from Sentry so they agree in case.
+ refreshPrivilegesInCatalog(sentryRole.getRoleName(), role, allRolesPrivileges);
}
return rolesToRemove;
}
@@ -211,6 +214,7 @@ public class SentryProxy {
// Assume all users should be removed. Then query the Policy Service and remove
// users from this set that actually exist.
Set<String> usersToRemove = catalog_.getAuthPolicy().getAllUserNames();
+ // The keys (user names) in listAllUsersPrivileges here are always in lower case.
Map<String, Set<TSentryPrivilege>> allUsersPrivileges =
sentryPolicyService_.listAllUsersPrivileges(processUser_);
for (Map.Entry<String, Set<TSentryPrivilege>> userPrivilegesEntry:
@@ -226,24 +230,33 @@ public class SentryProxy {
if (existingUser.getRef() && resetVersions_) {
user.setCatalogVersion(catalog_.incrementAndGetCatalogVersion());
}
- refreshPrivilegesInCatalog(user, allUsersPrivileges);
+ // allUsersPrivileges keys and userPrivilegesEntry.getKey() are used here since
+ // they both come from Sentry so they agree in case.
+ refreshPrivilegesInCatalog(userPrivilegesEntry.getKey(), user,
+ allUsersPrivileges);
}
return usersToRemove;
}
/**
* Updates the privileges for a given principal in the catalog since the last Sentry
- * sync update.
+ * sync update. The sentryPrincipalName is used to match against the key in
+ * allPrincipalPrivileges, which both come from Sentry, so they should have the
+ * same case sensitivity.
*/
- private void refreshPrivilegesInCatalog(Principal principal,
- Map<String, Set<TSentryPrivilege>> allPrincipalPrivileges)
+ private void refreshPrivilegesInCatalog(String sentryPrincipalName,
+ Principal principal, Map<String, Set<TSentryPrivilege>> allPrincipalPrivileges)
throws CatalogException {
// Assume all privileges should be removed. Privileges that still exist are
// deleted from this set and we are left with the set of privileges that need
// to be removed.
Set<String> privilegesToRemove = principal.getPrivilegeNames();
+ // It is important to get a set of privileges using sentryPrincipalName
+ // and not principal.getName() because principal.getName() may return a
+ // principal name with a different case than the principal names stored
+ // in allPrincipalPrivileges. See IMPALA-7729 for more information.
Set<TSentryPrivilege> sentryPrivileges = allPrincipalPrivileges.get(
- principal.getName());
+ sentryPrincipalName);
if (sentryPrivileges == null) return;
// Check all the privileges that are part of this principal.
for (TSentryPrivilege sentryPriv: sentryPrivileges) {
http://git-wip-us.apache.org/repos/asf/impala/blob/072f3ee9/tests/authorization/test_grant_revoke.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_grant_revoke.py b/tests/authorization/test_grant_revoke.py
index 480f2c5..949887b 100644
--- a/tests/authorization/test_grant_revoke.py
+++ b/tests/authorization/test_grant_revoke.py
@@ -369,3 +369,25 @@ class TestGrantRevoke(SentryCacheTestSuite):
assert privileges_before.data == privileges_after.data
finally:
self.client.execute("drop role {0}".format(role_name))
+
+ @pytest.mark.execute_serially
+ @SentryCacheTestSuite.with_args(
+ impalad_args="--server_name=server1",
+ catalogd_args="--sentry_config={0}".format(SENTRY_CONFIG_FILE),
+ sentry_config=SENTRY_CONFIG_FILE)
+ def test_invalidate_metadata(self, unique_role):
+ """IMPALA-7729: Tests running invalidate metadata with role names that have different
+ case sensitivity."""
+ for role_name in [unique_role.lower(), unique_role.upper(), unique_role.capitalize()]:
+ try:
+ self.client.execute("create role {0}".format(role_name))
+ self.client.execute("grant all on server to {0}".format(role_name))
+ self.client.execute("grant role {0} to group `{1}`".format(
+ role_name, grp.getgrnam(getuser()).gr_name))
+
+ # Verify that running invalidate metadata won't hang due to case sensitivity
+ # in the role names.
+ handle = self.client.execute_async("invalidate metadata")
+ assert self.client.wait_for_finished_timeout(handle, timeout=60)
+ finally:
+ self.client.execute("drop role {0}".format(role_name))
[5/5] impala git commit: IMPALA-7681. Add Azure Blob File System
(ADLS Gen2) support.
Posted by jo...@apache.org.
IMPALA-7681. Add Azure Blob File System (ADLS Gen2) support.
HADOOP-15407 adds a new FileSystem implementation called "ABFS" for the
ADLS Gen2 service. It's in the hadoop-azure module as a replacement for
WASB. Filesystem semantics should be the same, so skipped tests and
other behavior changes have simply mirrored what is done for ADLS Gen1
by default. Tests skipped on ADLS Gen1 due to eventual consistency of
the Python client can be run against ADLS Gen2.
Change-Id: I5120b071760e7655e78902dce8483f8f54de445d
Reviewed-on: http://gerrit.cloudera.org:8080/11630
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/7a022cf3
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/7a022cf3
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/7a022cf3
Branch: refs/heads/master
Commit: 7a022cf36a2c678dcff02d48db0641e6f74f068f
Parents: 0340a15
Author: Sean Mackrory <ma...@apache.org>
Authored: Tue Oct 9 15:10:36 2018 -0600
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sat Oct 20 06:43:00 2018 +0000
----------------------------------------------------------------------
be/src/exec/hdfs-table-sink.cc | 1 +
be/src/runtime/io/disk-io-mgr-test.cc | 3 +-
be/src/runtime/io/disk-io-mgr.cc | 7 ++
be/src/runtime/io/disk-io-mgr.h | 4 +
be/src/runtime/io/scan-range.cc | 8 +-
be/src/util/hdfs-util.cc | 8 ++
be/src/util/hdfs-util.h | 3 +
bin/impala-config.sh | 24 ++++
fe/pom.xml | 6 +
.../apache/impala/analysis/LoadDataStmt.java | 9 +-
.../org/apache/impala/catalog/HdfsTable.java | 1 +
.../apache/impala/common/FileSystemUtil.java | 24 ++++
.../org/apache/impala/service/JniFrontend.java | 4 +
.../impala/analysis/AnalyzeStmtsTest.java | 2 +-
.../common/etc/hadoop/conf/core-site.xml.tmpl | 25 ++++
tests/common/impala_test_suite.py | 6 +-
tests/common/skip.py | 22 ++++
.../custom_cluster/test_admission_controller.py | 2 +
tests/custom_cluster/test_hdfs_fd_caching.py | 7 +-
tests/custom_cluster/test_insert_behaviour.py | 3 +-
tests/custom_cluster/test_metadata_replicas.py | 2 +
.../test_parquet_max_page_header.py | 3 +-
tests/custom_cluster/test_permanent_udfs.py | 8 +-
tests/data_errors/test_data_errors.py | 5 +-
tests/failure/test_failpoints.py | 4 +-
tests/metadata/test_compute_stats.py | 3 +-
tests/metadata/test_ddl.py | 4 +-
tests/metadata/test_hdfs_encryption.py | 3 +-
tests/metadata/test_hdfs_permissions.py | 3 +-
tests/metadata/test_hms_integration.py | 4 +-
.../metadata/test_metadata_query_statements.py | 4 +-
tests/metadata/test_partition_metadata.py | 3 +-
tests/metadata/test_refresh_partition.py | 3 +-
tests/metadata/test_views_compatibility.py | 3 +-
tests/query_test/test_compressed_formats.py | 3 +-
tests/query_test/test_hdfs_caching.py | 7 +-
tests/query_test/test_insert.py | 4 +-
tests/query_test/test_insert_behaviour.py | 9 +-
tests/query_test/test_insert_parquet.py | 4 +-
tests/query_test/test_join_queries.py | 2 +
tests/query_test/test_nested_types.py | 3 +
tests/query_test/test_observability.py | 3 +-
tests/query_test/test_partitioning.py | 3 +-
tests/query_test/test_resource_limits.py | 5 +-
tests/query_test/test_scanners.py | 7 ++
tests/stress/test_ddl_stress.py | 3 +-
tests/util/abfs_util.py | 113 +++++++++++++++++++
tests/util/filesystem_utils.py | 5 +-
48 files changed, 360 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/exec/hdfs-table-sink.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-table-sink.cc b/be/src/exec/hdfs-table-sink.cc
index 9c46638..24df2ff 100644
--- a/be/src/exec/hdfs-table-sink.cc
+++ b/be/src/exec/hdfs-table-sink.cc
@@ -378,6 +378,7 @@ Status HdfsTableSink::CreateNewTmpFile(RuntimeState* state,
}
if (IsS3APath(output_partition->current_file_name.c_str()) ||
+ IsABFSPath(output_partition->current_file_name.c_str()) ||
IsADLSPath(output_partition->current_file_name.c_str())) {
// On S3A, the file cannot be stat'ed until after it's closed, and even so, the block
// size reported will be just the filesystem default. Similarly, the block size
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/runtime/io/disk-io-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc
index 7a9bc23..6afdb5d 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -51,6 +51,7 @@ DECLARE_int64(min_buffer_size);
DECLARE_int32(num_remote_hdfs_io_threads);
DECLARE_int32(num_s3_io_threads);
DECLARE_int32(num_adls_io_threads);
+DECLARE_int32(num_abfs_io_threads);
#ifndef NDEBUG
DECLARE_int32(stress_disk_read_delay_ms);
#endif
@@ -1565,7 +1566,7 @@ TEST_F(DiskIoMgrTest, ReadIntoClientBufferError) {
TEST_F(DiskIoMgrTest, VerifyNumThreadsParameter) {
InitRootReservation(LARGE_RESERVATION_LIMIT);
const int num_io_threads_for_remote_disks = FLAGS_num_remote_hdfs_io_threads
- + FLAGS_num_s3_io_threads + FLAGS_num_adls_io_threads;
+ + FLAGS_num_s3_io_threads + FLAGS_num_adls_io_threads + FLAGS_num_abfs_io_threads;
// Verify num_io_threads_per_rotational_disk and num_io_threads_per_solid_state_disk.
// Since we do not have control over which disk is used, we check for either type
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/runtime/io/disk-io-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index fa871d1..fb0c876 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -83,6 +83,8 @@ DEFINE_int32(num_remote_hdfs_io_threads, 8, "Number of remote HDFS I/O threads")
// open to S3 and use of multiple CPU cores since S3 reads are relatively compute
// expensive (SSL and JNI buffer overheads).
DEFINE_int32(num_s3_io_threads, 16, "Number of S3 I/O threads");
+// The maximum number of ABFS I/O threads. TODO: choose the default empirically.
+DEFINE_int32(num_abfs_io_threads, 16, "Number of ABFS I/O threads");
// The maximum number of ADLS I/O threads. This number is a good default to have for
// clusters that may vary widely in size, due to an undocumented concurrency limit
// enforced by ADLS for a cluster, which spans between 500-700. For smaller clusters
@@ -233,6 +235,9 @@ Status DiskIoMgr::Init() {
} else if (i == RemoteS3DiskId()) {
num_threads_per_disk = FLAGS_num_s3_io_threads;
device_name = "S3 remote";
+ } else if (i == RemoteAbfsDiskId()) {
+ num_threads_per_disk = FLAGS_num_abfs_io_threads;
+ device_name = "ABFS remote";
} else if (i == RemoteAdlsDiskId()) {
num_threads_per_disk = FLAGS_num_adls_io_threads;
device_name = "ADLS remote";
@@ -457,10 +462,12 @@ int DiskIoMgr::AssignQueue(const char* file, int disk_id, bool expected_local) {
return RemoteDfsDiskId();
}
if (IsS3APath(file)) return RemoteS3DiskId();
+ if (IsABFSPath(file)) return RemoteAbfsDiskId();
if (IsADLSPath(file)) return RemoteAdlsDiskId();
}
// Assign to a local disk queue.
DCHECK(!IsS3APath(file)); // S3 is always remote.
+ DCHECK(!IsABFSPath(file)); // ABFS is always remote.
DCHECK(!IsADLSPath(file)); // ADLS is always remote.
if (disk_id == -1) {
// disk id is unknown, assign it an arbitrary one.
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/runtime/io/disk-io-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.h b/be/src/runtime/io/disk-io-mgr.h
index d5c7643..d36906d 100644
--- a/be/src/runtime/io/disk-io-mgr.h
+++ b/be/src/runtime/io/disk-io-mgr.h
@@ -286,6 +286,9 @@ class DiskIoMgr : public CacheLineAligned {
/// The disk ID (and therefore disk_queues_ index) used for S3 accesses.
int RemoteS3DiskId() const { return num_local_disks() + REMOTE_S3_DISK_OFFSET; }
+ /// The disk ID (and therefore disk_queues_ index) used for ABFS accesses.
+ int RemoteAbfsDiskId() const { return num_local_disks() + REMOTE_ABFS_DISK_OFFSET; }
+
/// The disk ID (and therefore disk_queues_ index) used for ADLS accesses.
int RemoteAdlsDiskId() const { return num_local_disks() + REMOTE_ADLS_DISK_OFFSET; }
@@ -335,6 +338,7 @@ class DiskIoMgr : public CacheLineAligned {
REMOTE_DFS_DISK_OFFSET = 0,
REMOTE_S3_DISK_OFFSET,
REMOTE_ADLS_DISK_OFFSET,
+ REMOTE_ABFS_DISK_OFFSET,
REMOTE_NUM_DISKS
};
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/runtime/io/scan-range.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/scan-range.cc b/be/src/runtime/io/scan-range.cc
index 660710e..41915b3 100644
--- a/be/src/runtime/io/scan-range.cc
+++ b/be/src/runtime/io/scan-range.cc
@@ -29,11 +29,13 @@ using namespace impala;
using namespace impala::io;
// TODO: Run perf tests and empirically settle on the most optimal default value for the
-// read buffer size. Currently setting it as 128k for the same reason as for S3, i.e.
+// read buffer sizes. Currently setting them as 128k for the same reason as for S3, i.e.
// due to JNI array allocation and memcpy overhead, 128k was emperically found to have the
// least overhead.
DEFINE_int64(adls_read_chunk_size, 128 * 1024, "The maximum read chunk size to use when "
"reading from ADLS.");
+DEFINE_int64(abfs_read_chunk_size, 128 * 1024, "The maximum read chunk size to use when "
+ "reading from ABFS.");
// Implementation of the ScanRange functionality. Each ScanRange contains a queue
// of ready buffers. For each ScanRange, there is only a single producer and
@@ -554,6 +556,10 @@ int64_t ScanRange::MaxReadChunkSize() const {
DCHECK(IsADLSPath(file()));
return FLAGS_adls_read_chunk_size;
}
+ if (disk_id_ == io_mgr_->RemoteAbfsDiskId()) {
+ DCHECK(IsABFSPath(file()));
+ return FLAGS_abfs_read_chunk_size;
+ }
// The length argument of hdfsRead() is an int. Ensure we don't overflow it.
return numeric_limits<int>::max();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/util/hdfs-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/hdfs-util.cc b/be/src/util/hdfs-util.cc
index aa3466f..d344309 100644
--- a/be/src/util/hdfs-util.cc
+++ b/be/src/util/hdfs-util.cc
@@ -86,6 +86,14 @@ bool IsS3APath(const char* path) {
return strncmp(path, "s3a://", 6) == 0;
}
+bool IsABFSPath(const char* path) {
+ if (strstr(path, ":/") == NULL) {
+ return ExecEnv::GetInstance()->default_fs().compare(0, 7, "abfs://") == 0 ||
+ ExecEnv::GetInstance()->default_fs().compare(0, 8, "abfss://") == 0;
+ }
+ return strncmp(path, "abfs://", 7) == 0 || strncmp(path, "abfss://", 8) == 0;
+}
+
bool IsADLSPath(const char* path) {
if (strstr(path, ":/") == NULL) {
return ExecEnv::GetInstance()->default_fs().compare(0, 6, "adl://") == 0;
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/be/src/util/hdfs-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/hdfs-util.h b/be/src/util/hdfs-util.h
index 858a9b4..f57efd9 100644
--- a/be/src/util/hdfs-util.h
+++ b/be/src/util/hdfs-util.h
@@ -50,6 +50,9 @@ bool IsHdfsPath(const char* path);
/// Returns true iff the path refers to a location on an S3A filesystem.
bool IsS3APath(const char* path);
+/// Returns true iff the path refers to a location on an ABFS filesystem.
+bool IsABFSPath(const char* path);
+
/// Returns true iff the path refers to a location on an ADL filesystem.
bool IsADLSPath(const char* path);
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 9f8408c..57538e5 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -263,6 +263,8 @@ export azure_tenant_id="${azure_tenant_id-DummyAdlsTenantId}"
export azure_client_id="${azure_client_id-DummyAdlsClientId}"
export azure_client_secret="${azure_client_secret-DummyAdlsClientSecret}"
export azure_data_lake_store_name="${azure_data_lake_store_name-}"
+export azure_storage_account_name="${azure_storage_account_name-}"
+export azure_storage_container_name="${azure_storage_container_name-}"
export HDFS_REPLICATION="${HDFS_REPLICATION-3}"
export ISILON_NAMENODE="${ISILON_NAMENODE-}"
export DEFAULT_FS="${DEFAULT_FS-hdfs://localhost:20500}"
@@ -352,6 +354,28 @@ elif [ "${TARGET_FILESYSTEM}" = "adls" ]; then
fi
DEFAULT_FS="adl://${azure_data_lake_store_name}.azuredatalakestore.net"
export DEFAULT_FS
+elif [ "${TARGET_FILESYSTEM}" = "abfs" ]; then
+ # ABFS is also known as ADLS Gen2, and they can share credentials
+ # Basic error checking
+ if [[ "${azure_client_id}" = "DummyAdlsClientId" ||\
+ "${azure_tenant_id}" = "DummyAdlsTenantId" ||\
+ "${azure_client_secret}" = "DummyAdlsClientSecret" ]]; then
+ echo "All 3 of the following need to be assigned valid values and belong
+ to the owner of the Azure storage account in order to access the
+ filesystem: azure_client_id, azure_tenant_id, azure_client_secret."
+ return 1
+ fi
+ if [[ "${azure_storage_account_name}" = "" ]]; then
+ echo "azure_storage_account_name cannot be an empty string for ABFS"
+ return 1
+ fi
+ if [[ "${azure_storage_container_name}" = "" ]]; then
+ echo "azure_storage_container_name cannot be an empty string for ABFS"
+ return 1
+ fi
+ domain="${azure_storage_account_name}.dfs.core.windows.net"
+ DEFAULT_FS="abfss://${azure_storage_container_name}@${domain}"
+ export DEFAULT_FS
elif [ "${TARGET_FILESYSTEM}" = "isilon" ]; then
if [ "${ISILON_NAMENODE}" = "" ]; then
echo "In order to access the Isilon filesystem, ISILON_NAMENODE"
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/pom.xml
----------------------------------------------------------------------
diff --git a/fe/pom.xml b/fe/pom.xml
index 5199d4f..f44c14f 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -88,6 +88,12 @@ under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure-datalake</artifactId>
<version>${hadoop.version}</version>
<exclusions>
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
index 128973f..0608987 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
@@ -24,6 +24,8 @@ import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -143,9 +145,11 @@ public class LoadDataStmt extends StatementBase {
Path source = sourceDataPath_.getPath();
FileSystem fs = source.getFileSystem(FileSystemUtil.getConfiguration());
if (!(fs instanceof DistributedFileSystem) && !(fs instanceof S3AFileSystem) &&
+ !(fs instanceof AzureBlobFileSystem) &&
+ !(fs instanceof SecureAzureBlobFileSystem) &&
!(fs instanceof AdlFileSystem)) {
throw new AnalysisException(String.format("INPATH location '%s' " +
- "must point to an HDFS, S3A or ADL filesystem.", sourceDataPath_));
+ "must point to an HDFS, S3A, ADL or ABFS filesystem.", sourceDataPath_));
}
if (!fs.exists(source)) {
throw new AnalysisException(String.format(
@@ -157,7 +161,8 @@ public class LoadDataStmt extends StatementBase {
// its parent directory (in order to delete the file as part of the move operation).
FsPermissionChecker checker = FsPermissionChecker.getInstance();
// TODO: Disable permission checking for S3A as well (HADOOP-13892)
- boolean shouldCheckPerms = !(fs instanceof AdlFileSystem);
+ boolean shouldCheckPerms = !(fs instanceof AdlFileSystem ||
+ fs instanceof AzureBlobFileSystem || fs instanceof SecureAzureBlobFileSystem);
if (fs.isDirectory(source)) {
if (FileSystemUtil.getTotalNumVisibleFiles(source) == 0) {
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 4eba255..1087e87 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -918,6 +918,7 @@ public class HdfsTable extends Table implements FeFsTable {
// behavior. So ADLS ACLs are unsupported until the connector is able to map
// permissions to hadoop users/groups (HADOOP-14437).
if (FileSystemUtil.isADLFileSystem(fs)) return TAccessLevel.READ_WRITE;
+ if (FileSystemUtil.isABFSFileSystem(fs)) return TAccessLevel.READ_WRITE;
while (location != null) {
try {
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index c237426..43d164b 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
@@ -306,6 +308,7 @@ public class FileSystemUtil {
if (isDistributedFileSystem(fs)) return true;
// Blacklist FileSystems that are known to not to include storage UUIDs.
return !(fs instanceof S3AFileSystem || fs instanceof LocalFileSystem ||
+ fs instanceof AzureBlobFileSystem || fs instanceof SecureAzureBlobFileSystem ||
fs instanceof AdlFileSystem);
}
@@ -338,6 +341,26 @@ public class FileSystemUtil {
}
/**
+ * Returns true iff the filesystem is AzureBlobFileSystem or
+ * SecureAzureBlobFileSystem. This function is unique in that there are 2
+ * distinct classes it checks for, but the ony functional difference is the
+ * use of wire encryption. Some features like OAuth authentication do require
+ * wire encryption but that does not matter in usages of this function.
+ */
+ public static boolean isABFSFileSystem(FileSystem fs) {
+ return fs instanceof AzureBlobFileSystem
+ || fs instanceof SecureAzureBlobFileSystem;
+ }
+
+ /**
+ * Returns true iff the path is on AzureBlobFileSystem or
+ * SecureAzureBlobFileSystem.
+ */
+ public static boolean isABFSFileSystem(Path path) throws IOException {
+ return isABFSFileSystem(path.getFileSystem(CONF));
+ }
+
+ /**
* Returns true iff the filesystem is an instance of LocalFileSystem.
*/
public static boolean isLocalFileSystem(FileSystem fs) {
@@ -458,6 +481,7 @@ public class FileSystemUtil {
return (FileSystemUtil.isDistributedFileSystem(path) ||
FileSystemUtil.isLocalFileSystem(path) ||
FileSystemUtil.isS3AFileSystem(path) ||
+ FileSystemUtil.isABFSFileSystem(path) ||
FileSystemUtil.isADLFileSystem(path));
}
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 0a066d0..db2ad29 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -799,6 +801,8 @@ public class JniFrontend {
FileSystem fs = FileSystem.get(CONF);
if (!(fs instanceof DistributedFileSystem ||
fs instanceof S3AFileSystem ||
+ fs instanceof AzureBlobFileSystem ||
+ fs instanceof SecureAzureBlobFileSystem ||
fs instanceof AdlFileSystem)) {
return "Currently configured default filesystem: " +
fs.getClass().getSimpleName() + ". " +
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index d36b97d..4fbffab 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -3204,7 +3204,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalysisError(String.format("load data inpath '%s' %s into table " +
"tpch.lineitem", "file:///test-warehouse/test.out", overwrite),
"INPATH location 'file:/test-warehouse/test.out' must point to an " +
- "HDFS, S3A or ADL filesystem.");
+ "HDFS, S3A, ADL or ABFS filesystem.");
// File type / table type mismatch.
AnalyzesOk(String.format("load data inpath '%s' %s into table " +
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl
index b11967a..982b919 100644
--- a/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/core-site.xml.tmpl
@@ -110,6 +110,31 @@ DEFAULT</value>
<value>https://login.windows.net/${azure_tenant_id}/oauth2/token</value>
</property>
+ <property>
+ <name>fs.azure.account.auth.type</name>
+ <value>OAuth</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth.provider.type</name>
+ <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.id</name>
+ <value>${azure_client_id}</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.secret</name>
+ <value>${azure_client_secret}</value>
+ </property>
+
+ <property>
+ <name>fs.azure.account.oauth2.client.endpoint</name>
+ <value>https://login.microsoftonline.com/${azure_tenant_id}/oauth2/token</value>
+ </property>
+
<!-- This property can be used in tests to ascertain that this core-site.xml from
the classpath has been loaded. (Ex: TestRequestPoolService) -->
<property>
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 954ca4c..f4f0c03 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -56,6 +56,7 @@ from tests.performance.query_exec_functions import execute_using_jdbc
from tests.performance.query_executor import JdbcQueryExecConfig
from tests.util.filesystem_utils import (
IS_S3,
+ IS_ABFS,
IS_ADLS,
S3_BUCKET_NAME,
ADLS_STORE_NAME,
@@ -67,6 +68,7 @@ from tests.util.hdfs_util import (
get_hdfs_client_from_conf,
NAMENODE)
from tests.util.s3_util import S3Client
+from tests.util.abfs_util import ABFSClient
from tests.util.test_file_parser import (
QueryTestSectionReader,
parse_query_test_file,
@@ -162,6 +164,8 @@ class ImpalaTestSuite(BaseTestSuite):
cls.filesystem_client = cls.hdfs_client
if IS_S3:
cls.filesystem_client = S3Client(S3_BUCKET_NAME)
+ elif IS_ABFS:
+ cls.filesystem_client = ABFSClient()
elif IS_ADLS:
cls.filesystem_client = ADLSClient(ADLS_STORE_NAME)
@@ -789,7 +793,7 @@ class ImpalaTestSuite(BaseTestSuite):
# If 'skip_hbase' is specified or the filesystem is isilon, s3 or local, we don't
# need the hbase dimension.
if pytest.config.option.skip_hbase or TARGET_FILESYSTEM.lower() \
- in ['s3', 'isilon', 'local', 'adls']:
+ in ['s3', 'isilon', 'local', 'abfs', 'adls']:
for tf_dimension in tf_dimensions:
if tf_dimension.value.file_format == "hbase":
tf_dimensions.remove(tf_dimension)
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/common/skip.py
----------------------------------------------------------------------
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 6143fee..2f19bf4 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -26,6 +26,7 @@ from functools import partial
from tests.common.environ import IMPALAD_BUILD
from tests.util.filesystem_utils import (
+ IS_ABFS,
IS_ADLS,
IS_EC,
IS_HDFS,
@@ -55,6 +56,27 @@ class SkipIfS3:
qualified_path = pytest.mark.skipif(IS_S3,
reason="Tests rely on HDFS qualified paths, IMPALA-1872")
+
+class SkipIfABFS:
+
+ # These ones are skipped due to product limitations.
+ caching = pytest.mark.skipif(IS_ABFS, reason="SET CACHED not implemented for ABFS")
+ hive = pytest.mark.skipif(IS_ABFS, reason="Hive doesn't work with ABFS")
+ hdfs_block_size = pytest.mark.skipif(IS_ABFS, reason="ABFS uses it's own block size")
+ hdfs_acls = pytest.mark.skipif(IS_ABFS, reason="HDFS acls are not supported on ABFS")
+ jira = partial(pytest.mark.skipif, IS_ABFS)
+ hdfs_encryption = pytest.mark.skipif(IS_ABFS,
+ reason="HDFS encryption is not supported with ABFS")
+ trash = pytest.mark.skipif(IS_ABFS,
+ reason="Drop/purge not working as expected on ABFS, IMPALA-7726")
+
+ # These ones need test infra work to re-enable.
+ udfs = pytest.mark.skipif(IS_ABFS, reason="udas/udfs not copied to ABFS")
+ datasrc = pytest.mark.skipif(IS_ABFS, reason="data sources not copied to ABFS")
+ hbase = pytest.mark.skipif(IS_ABFS, reason="HBase not started with ABFS")
+ qualified_path = pytest.mark.skipif(IS_ABFS,
+ reason="Tests rely on HDFS qualified paths, IMPALA-1872")
+
class SkipIfADLS:
# These ones are skipped due to product limitations.
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index fc74f00..018f66d 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -36,6 +36,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.resource_pool_config import ResourcePoolConfig
from tests.common.skip import (
SkipIfS3,
+ SkipIfABFS,
SkipIfADLS,
SkipIfEC,
SkipIfNotHdfsMinicluster)
@@ -439,6 +440,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
".* is greater than pool max mem resources 10.00 MB", str(ex))
@SkipIfS3.hdfs_block_size
+ @SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfEC.fix_later
@pytest.mark.execute_serially
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_hdfs_fd_caching.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index 1afe431..8849363 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -22,6 +22,7 @@ from tests.common.skip import SkipIfLocal, SkipIfEC
from tests.util.filesystem_utils import (
IS_ISILON,
IS_S3,
+ IS_ABFS,
IS_ADLS)
from time import sleep
@@ -132,7 +133,8 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
# Caching only applies to local HDFS files. If this is local HDFS, then verify
# that caching works. Otherwise, verify that file handles are not cached.
- if IS_S3 or IS_ADLS or IS_ISILON or pytest.config.option.testing_remote_cluster:
+ if IS_S3 or IS_ABFS or IS_ADLS or IS_ISILON or \
+ pytest.config.option.testing_remote_cluster:
caching_expected = False
else:
caching_expected = True
@@ -148,7 +150,8 @@ class TestHdfsFdCaching(CustomClusterTestSuite):
handle_timeout = 5
# Only test eviction on platforms where caching is enabled.
- if IS_S3 or IS_ADLS or IS_ISILON or pytest.config.option.testing_remote_cluster:
+ if IS_S3 or IS_ABFS or IS_ADLS or IS_ISILON or \
+ pytest.config.option.testing_remote_cluster:
return
caching_expected = True
self.run_fd_caching_test(vector, caching_expected, cache_capacity, handle_timeout)
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_insert_behaviour.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_insert_behaviour.py b/tests/custom_cluster/test_insert_behaviour.py
index af6a270..7d3fb3e 100644
--- a/tests/custom_cluster/test_insert_behaviour.py
+++ b/tests/custom_cluster/test_insert_behaviour.py
@@ -18,13 +18,14 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.util.filesystem_utils import IS_ISILON, WAREHOUSE
from tests.util.hdfs_util import HdfsConfig, get_hdfs_client, get_hdfs_client_from_conf
TEST_TBL = "insert_inherit_permission"
@SkipIfS3.hdfs_acls
+@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_metadata_replicas.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py
index 9674756..940f371 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -22,12 +22,14 @@ from tests.common.environ import specific_build_type_timeout
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import (
SkipIfS3,
+ SkipIfABFS,
SkipIfADLS,
SkipIfIsilon,
SkipIfLocal)
from tests.util.hive_utils import HiveDbWrapper
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_parquet_max_page_header.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index 913d883..59e87fb 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -24,7 +24,7 @@ import string
import subprocess
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon
class TestParquetMaxPageHeader(CustomClusterTestSuite):
'''This tests large page headers in parquet files. Parquet page header size can
@@ -92,6 +92,7 @@ class TestParquetMaxPageHeader(CustomClusterTestSuite):
put.wait()
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@pytest.mark.execute_serially
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/custom_cluster/test_permanent_udfs.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_permanent_udfs.py b/tests/custom_cluster/test_permanent_udfs.py
index 100b5ea..34a1e00 100644
--- a/tests/custom_cluster/test_permanent_udfs.py
+++ b/tests/custom_cluster/test_permanent_udfs.py
@@ -24,7 +24,7 @@ import subprocess
from tempfile import mkdtemp
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.filesystem_utils import get_fs_path
@@ -162,6 +162,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@pytest.mark.execute_serially
@@ -183,6 +184,7 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
@pytest.mark.execute_serially
@@ -246,6 +248,8 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
+ @SkipIfADLS.hive
@SkipIfLocal.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
@@ -307,6 +311,8 @@ class TestUdfPersistence(CustomClusterTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
+ @SkipIfADLS.hive
@SkipIfLocal.hive
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/data_errors/test_data_errors.py
----------------------------------------------------------------------
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index d8c662a..52e921f 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -25,7 +25,7 @@ import subprocess
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIf, SkipIfS3, SkipIfADLS, SkipIfLocal
+from tests.common.skip import SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import create_exec_option_dimension
class TestDataErrors(ImpalaTestSuite):
@@ -106,6 +106,7 @@ class TestHdfsUnknownErrors(ImpalaTestSuite):
assert "Safe mode is OFF" in output
@SkipIfS3.qualified_path
+@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
class TestHdfsScanNodeErrors(TestDataErrors):
@classmethod
@@ -124,6 +125,7 @@ class TestHdfsScanNodeErrors(TestDataErrors):
self.run_test_case('DataErrorsTest/hdfs-scan-node-errors', vector)
@SkipIfS3.qualified_path
+@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
@SkipIfLocal.qualified_path
class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
@@ -139,6 +141,7 @@ class TestHdfsSeqScanNodeErrors(TestHdfsScanNodeErrors):
@SkipIfS3.qualified_path
+@SkipIfABFS.qualified_path
@SkipIfADLS.qualified_path
class TestHdfsRcFileScanNodeErrors(TestHdfsScanNodeErrors):
@classmethod
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/failure/test_failpoints.py
----------------------------------------------------------------------
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index ef9ed07..d6cc1da 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -26,7 +26,8 @@ from time import sleep
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
-from tests.common.skip import SkipIf, SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIf, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, \
+ SkipIfLocal
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
@@ -53,6 +54,7 @@ QUERIES = [
@SkipIf.skip_hbase # -skip_hbase argument specified
@SkipIfS3.hbase # S3: missing coverage: failures
+@SkipIfABFS.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase # ISILON: missing coverage: failures.
@SkipIfLocal.hbase
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_compute_stats.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index c3d182d..9adc8cd 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -20,7 +20,7 @@ from subprocess import check_call
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_single_exec_option_dimension,
@@ -71,6 +71,7 @@ class TestComputeStats(ImpalaTestSuite):
self.cleanup_db("parquet")
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 40ec569..b510af1 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -25,7 +25,7 @@ from test_ddl_base import TestDdlBase
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import LOG
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIf, SkipIfADLS, SkipIfLocal
+from tests.common.skip import SkipIf, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import WAREHOUSE, IS_HDFS, IS_S3, IS_ADLS
from tests.common.impala_cluster import ImpalaCluster
@@ -33,6 +33,7 @@ from tests.common.impala_cluster import ImpalaCluster
# Validates DDL statements (create, drop)
class TestDdlStatements(TestDdlBase):
@SkipIfLocal.hdfs_client
+ @SkipIfABFS.trash
def test_drop_table_with_purge(self, unique_database):
"""This test checks if the table data is permamently deleted in
DROP TABLE <tbl> PURGE queries"""
@@ -425,6 +426,7 @@ class TestDdlStatements(TestDdlBase):
use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
@SkipIfLocal.hdfs_client
+ @SkipIfABFS.trash
def test_drop_partition_with_purge(self, vector, unique_database):
"""Verfies whether alter <tbl> drop partition purge actually skips trash"""
self.client.execute(
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_hdfs_encryption.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_hdfs_encryption.py b/tests/metadata/test_hdfs_encryption.py
index 27eaaee..fef4ac1 100644
--- a/tests/metadata/test_hdfs_encryption.py
+++ b/tests/metadata/test_hdfs_encryption.py
@@ -19,7 +19,7 @@ import getpass
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -34,6 +34,7 @@ TMP_DIR = '/%s' % (PYWEBHDFS_TMP_DIR)
@SkipIfS3.hdfs_encryption
+@SkipIfABFS.hdfs_encryption
@SkipIfADLS.hdfs_encryption
@SkipIfIsilon.hdfs_encryption
@SkipIfLocal.hdfs_encryption
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_hdfs_permissions.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_hdfs_permissions.py b/tests/metadata/test_hdfs_permissions.py
index a49b56e..d495fc4 100644
--- a/tests/metadata/test_hdfs_permissions.py
+++ b/tests/metadata/test_hdfs_permissions.py
@@ -16,7 +16,7 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -28,6 +28,7 @@ TBL_LOC = '%s/%s' % (WAREHOUSE, TEST_TBL)
@SkipIfS3.hdfs_acls
+@SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfLocal.hdfs_client
class TestHdfsPermissions(ImpalaTestSuite):
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_hms_integration.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 3ac5cef..800799c 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -30,13 +30,14 @@ import string
from subprocess import call
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
from tests.util.hive_utils import HiveDbWrapper, HiveTableWrapper
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
@@ -84,6 +85,7 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
assert 'test_tbl' in self.client.execute("show tables in hms_sanity_db").data
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_metadata_query_statements.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index bf21ecb..81fd60d 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -22,7 +22,7 @@ import re
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfIsilon, SkipIfS3, SkipIfADLS, SkipIfLocal
+from tests.common.skip import SkipIfIsilon, SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfLocal
from tests.common.test_dimensions import ALL_NODES_ONLY
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -75,6 +75,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
# data doesn't reside in hdfs.
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
def test_describe_formatted(self, vector, unique_database):
@@ -149,6 +150,7 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
self.client.execute(self.CREATE_DATA_SRC_STMT % (name,))
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_partition_metadata.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_partition_metadata.py b/tests/metadata/test_partition_metadata.py
index 302b0c9..280b7d0 100644
--- a/tests/metadata/test_partition_metadata.py
+++ b/tests/metadata/test_partition_metadata.py
@@ -17,7 +17,7 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import (create_single_exec_option_dimension,
create_uncompressed_text_dimension)
from tests.util.filesystem_utils import get_fs_path, WAREHOUSE, FILESYSTEM_PREFIX
@@ -89,6 +89,7 @@ class TestPartitionMetadata(ImpalaTestSuite):
assert data.split('\t') == ['6', '9']
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_refresh_partition.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index 2c31b93..4b2c67b 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -17,11 +17,12 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.util.filesystem_utils import get_fs_path
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/metadata/test_views_compatibility.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index ac208d7..a0103e2 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -23,7 +23,7 @@ from subprocess import call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader
@@ -47,6 +47,7 @@ from tests.util.test_file_parser import QueryTestSectionReader
# Missing Coverage: Views created by Hive and Impala being visible and queryble by each
# other on non hdfs storage.
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_compressed_formats.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_compressed_formats.py b/tests/query_test/test_compressed_formats.py
index 2896632..97a8782 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -23,7 +23,7 @@ from os.path import join
from subprocess import call
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import get_fs_path
@@ -40,6 +40,7 @@ compression_formats = [
# Missing Coverage: Compressed data written by Hive is queriable by Impala on a non-hdfs
# filesystem.
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_hdfs_caching.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 16940a0..176fe09 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -25,13 +25,15 @@ from subprocess import check_call
from tests.common.environ import specific_build_type_timeout
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal, SkipIfEC
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, \
+ SkipIfLocal, SkipIfEC
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
from tests.util.shell_util import exec_process
# End to end test that hdfs caching is working.
@SkipIfS3.caching # S3: missing coverage: verify SET CACHED gives error
+@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@SkipIfLocal.caching
@@ -109,6 +111,7 @@ class TestHdfsCaching(ImpalaTestSuite):
# run as a part of exhaustive tests which require the workload to be 'functional-query'.
# TODO: Move this to TestHdfsCaching once we make exhaustive tests run for other workloads
@SkipIfS3.caching
+@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@SkipIfLocal.caching
@@ -118,6 +121,7 @@ class TestHdfsCachingFallbackPath(ImpalaTestSuite):
return 'functional-query'
@SkipIfS3.hdfs_encryption
+ @SkipIfABFS.hdfs_encryption
@SkipIfADLS.hdfs_encryption
@SkipIfIsilon.hdfs_encryption
@SkipIfLocal.hdfs_encryption
@@ -169,6 +173,7 @@ class TestHdfsCachingFallbackPath(ImpalaTestSuite):
@SkipIfS3.caching
+@SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@SkipIfLocal.caching
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_insert.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 46d5249..8ded894 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -22,7 +22,7 @@ import pytest
from testdata.common import widetable
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfEC, SkipIfLocal, SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, SkipIfNotHdfsMinicluster
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -112,6 +112,8 @@ class TestInsertQueries(ImpalaTestSuite):
@pytest.mark.execute_serially
# Erasure coding doesn't respect memory limit
@SkipIfEC.fix_later
+ # ABFS partition names cannot end in periods
+ @SkipIfABFS.jira(reason="HADOOP-15860")
def test_insert(self, vector):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_insert_behaviour.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index e42d33a..bd0dd4e 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -23,7 +23,7 @@ import re
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_S3
@SkipIfLocal.hdfs_client
@@ -132,6 +132,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
assert len(self.filesystem_client.ls(part_dir)) == 1
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
@pytest.mark.xfail(run=False, reason="Fails intermittently on test clusters")
@@ -193,6 +194,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
check_has_acls("p1=1/p2=2/p3=30", "default:group:new_leaf_group:-w-")
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_insert_file_permissions(self, unique_database):
@@ -244,6 +246,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.execute_query_expect_success(self.client, insert_query)
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_mixed_partition_permissions(self, unique_database):
@@ -324,6 +327,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
load_data(self.execute_query_expect_success, "added_part")
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_readonly_table_dir(self, unique_database):
@@ -355,6 +359,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
assert re.search(r'Impala does not have WRITE access.*' + table_path, str(err))
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_insert_acl_permissions(self, unique_database):
@@ -433,6 +438,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.execute_query_expect_success(self.client, insert_query)
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_load_permissions(self, unique_database):
@@ -557,6 +563,7 @@ class TestInsertBehaviour(ImpalaTestSuite):
self.execute_query_expect_failure(self.client, insert_query)
@SkipIfS3.hdfs_acls
+ @SkipIfABFS.hdfs_acls
@SkipIfADLS.hdfs_acls
@SkipIfIsilon.hdfs_acls
def test_multiple_group_acls(self, unique_database):
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_insert_parquet.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 4523b1c..0bf28fe 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -29,7 +29,8 @@ from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder
from tests.common.environ import impalad_basedir
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIfEC, SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfADLS
+from tests.common.skip import SkipIfEC, SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfABFS, \
+ SkipIfADLS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import get_fs_path
@@ -360,6 +361,7 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfLocal.hive
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
# TODO: Should we move this to test_parquet_stats.py?
class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_join_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py
index 30c659c..51020c7 100644
--- a/tests/query_test/test_join_queries.py
+++ b/tests/query_test/test_join_queries.py
@@ -26,6 +26,7 @@ from tests.common.skip import (
SkipIfIsilon,
SkipIfLocal,
SkipIfS3,
+ SkipIfABFS,
SkipIfADLS)
from tests.common.test_vector import ImpalaTestDimension
@@ -62,6 +63,7 @@ class TestJoinQueries(ImpalaTestSuite):
self.run_test_case('QueryTest/single-node-joins-with-limits-exhaustive', new_vector)
@SkipIfS3.hbase
+ @SkipIfABFS.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase
@SkipIf.skip_hbase
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_nested_types.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 0603745..4fccd9d 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -26,6 +26,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (
SkipIfIsilon,
SkipIfS3,
+ SkipIfABFS,
SkipIfADLS,
SkipIfEC,
SkipIfLocal,
@@ -95,6 +96,7 @@ class TestNestedTypes(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
def test_upper_case_field_name(self, unique_database):
@@ -568,6 +570,7 @@ class TestMaxNestingDepth(ImpalaTestSuite):
@SkipIfIsilon.hive
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfLocal.hive
def test_load_hive_table(self, vector, unique_database):
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_observability.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 7e5c757..c351e02 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -17,7 +17,7 @@
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.util.filesystem_utils import IS_EC
import logging
import pytest
@@ -63,6 +63,7 @@ class TestObservability(ImpalaTestSuite):
@SkipIfS3.hbase
@SkipIfLocal.hbase
@SkipIfIsilon.hbase
+ @SkipIfABFS.hbase
@SkipIfADLS.hbase
def test_scan_summary(self):
"""IMPALA-4499: Checks that the exec summary for scans show the table name."""
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_partitioning.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 267d9e0..bb554ed 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -20,7 +20,7 @@ import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
# Tests to validate HDFS partitioning.
@@ -47,6 +47,7 @@ class TestPartitioning(ImpalaTestSuite):
# Missing Coverage: Impala deals with boolean partitions created by Hive on a non-hdfs
# filesystem.
@SkipIfS3.hive
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_resource_limits.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_resource_limits.py b/tests/query_test/test_resource_limits.py
index 58ae4a5..265421d 100644
--- a/tests/query_test/test_resource_limits.py
+++ b/tests/query_test/test_resource_limits.py
@@ -16,7 +16,8 @@
# under the License.
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfEC, SkipIfKudu, SkipIfLocal, SkipIfS3
+from tests.common.skip import SkipIfEC, SkipIfKudu, SkipIfLocal, SkipIfS3, SkipIfABFS, \
+ SkipIfADLS
from tests.common.test_dimensions import create_parquet_dimension
@@ -45,6 +46,8 @@ class TestResourceLimits(ImpalaTestSuite):
self.run_test_case('QueryTest/query-resource-limits', vector)
@SkipIfS3.hbase
+ @SkipIfADLS.hbase
+ @SkipIfABFS.hbase
@SkipIfLocal.multiple_impalad
def test_resource_limits_hbase(self, vector):
self.run_test_case('QueryTest/query-resource-limits-hbase', vector)
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 0000867..b75c3cc 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -35,6 +35,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import (
SkipIf,
SkipIfS3,
+ SkipIfABFS,
SkipIfADLS,
SkipIfEC,
SkipIfIsilon,
@@ -351,6 +352,7 @@ class TestParquet(ImpalaTestSuite):
assert len(result.data) == 1
assert "4294967294" in result.data
+ @SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
@@ -473,6 +475,7 @@ class TestParquet(ImpalaTestSuite):
vector, unique_database)
@SkipIfS3.hdfs_block_size
+ @SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
@@ -530,6 +533,7 @@ class TestParquet(ImpalaTestSuite):
assert total == num_scanners_with_no_reads
@SkipIfS3.hdfs_block_size
+ @SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
@@ -545,6 +549,7 @@ class TestParquet(ImpalaTestSuite):
self._multiple_blocks_helper(table_name, 40000, ranges_per_node=2)
@SkipIfS3.hdfs_block_size
+ @SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
@@ -906,6 +911,7 @@ class TestTextScanRangeLengths(ImpalaTestSuite):
# Missing Coverage: No coverage for truncated files errors or scans.
@SkipIfS3.hive
+@SkipIfABFS.hive
@SkipIfADLS.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
@@ -984,6 +990,7 @@ class TestOrc(ImpalaTestSuite):
lambda v: v.get_value('table_format').file_format == 'orc')
@SkipIfS3.hdfs_block_size
+ @SkipIfABFS.hdfs_block_size
@SkipIfADLS.hdfs_block_size
@SkipIfEC.fix_later
@SkipIfIsilon.hdfs_block_size
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/stress/test_ddl_stress.py
----------------------------------------------------------------------
diff --git a/tests/stress/test_ddl_stress.py b/tests/stress/test_ddl_stress.py
index b46f201..ea077a6 100644
--- a/tests/stress/test_ddl_stress.py
+++ b/tests/stress/test_ddl_stress.py
@@ -18,7 +18,7 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
# Number of tables to create per thread
NUM_TBLS_PER_THREAD = 10
@@ -48,6 +48,7 @@ class TestDdlStress(ImpalaTestSuite):
v.get_value('table_format').compression_codec == 'none'))
@SkipIfS3.caching
+ @SkipIfABFS.caching
@SkipIfADLS.caching
@SkipIfIsilon.caching
@SkipIfLocal.caching
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/util/abfs_util.py
----------------------------------------------------------------------
diff --git a/tests/util/abfs_util.py b/tests/util/abfs_util.py
new file mode 100644
index 0000000..8567888
--- /dev/null
+++ b/tests/util/abfs_util.py
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ABFS access utilities
+#
+# This file uses the Hadoop CLI to provide simple functions to the Impala test
+# suite to whatever the default filesystem is
+
+import re
+import subprocess
+import tempfile
+
+from tests.util.filesystem_base import BaseFilesystem
+
+
+class ABFSClient(BaseFilesystem):
+
+ def _hadoop_fs_shell(self, command):
+ hadoop_command = ['hadoop', 'fs'] + command
+ process = subprocess.Popen(hadoop_command,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ status = process.returncode
+ return (status, stdout, stderr)
+
+ def create_file(self, path, file_data, overwrite=True):
+ fixed_path = self._normalize_path(path)
+ if not overwrite and self.exists(fixed_path): return False
+ f = tempfile.NamedTemporaryFile(delete=False)
+ tmp_path = f.name
+ f.write(file_data)
+ f.close()
+ (status, stdout, stderr) = \
+ self._hadoop_fs_shell(['-put', tmp_path, fixed_path])
+ return status == 0
+
+ def make_dir(self, path, permission=None):
+ fixed_path = self._normalize_path(path)
+ self._hadoop_fs_shell(['-mkdir', '-p', fixed_path])
+ return True
+
+ def copy(self, src, dst):
+ fixed_src = self._normalize_path(src)
+ fixed_dst = self._normalize_path(dst)
+ (status, stdout, stderr) = \
+ self._hadoop_fs_shell(['-cp', fixed_src, fixed_dst])
+ assert status == 0, \
+ 'ABFS copy failed: ' + stderr + "; " + stdout
+ assert self.exists(dst), \
+ 'ABFS copy failed: Destination file {dst} does not exist'\
+ .format(dst=dst)
+
+ def _inner_ls(self, path):
+ fixed_path = self._normalize_path(path)
+ (status, stdout, stderr) = self._hadoop_fs_shell(['-ls', fixed_path])
+ # Trim the "Found X items" line and trailing new-line
+ entries = stdout.split("\n")[1:-1]
+ files = []
+ for entry in entries:
+ fields = re.split(" +", entry)
+ files.append({
+ 'name': fields[7],
+ 'length': int(fields[4]),
+ 'mode': fields[0]
+ })
+ return files
+
+ def ls(self, path):
+ fixed_path = self._normalize_path(path)
+ files = []
+ for f in self._inner_ls(fixed_path):
+ fname = f['name'].split("/")[-1]
+ if not fname == '':
+ files += [fname]
+ return files
+
+ def exists(self, path):
+ fixed_path = self._normalize_path(path)
+ (status, stdout, stderr) = self._hadoop_fs_shell(['-test', '-e', fixed_path])
+ return status == 0
+
+ def delete_file_dir(self, path, recursive=False):
+ fixed_path = self._normalize_path(path)
+ rm_command = ['-rm', fixed_path]
+ if recursive:
+ rm_command = ['-rm', '-r', fixed_path]
+ (status, stdout, stderr) = self._hadoop_fs_shell(rm_command)
+ return status == 0
+
+ def get_all_file_sizes(self, path):
+ """Returns a list of integers which are all the file sizes of files found
+ under 'path'."""
+ fixed_path = self._normalize_path(path)
+ return [f['length'] for f in
+ self._inner_ls(fixed_path) if f['mode'][0] == "-"]
+
+ def _normalize_path(self, path):
+ # Paths passed in may lack a leading slash
+ return path if path.startswith('/') else '/' + path
http://git-wip-us.apache.org/repos/asf/impala/blob/7a022cf3/tests/util/filesystem_utils.py
----------------------------------------------------------------------
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index aef08a3..a7b0809 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -30,6 +30,7 @@ IS_ISILON = FILESYSTEM == "isilon"
IS_LOCAL = FILESYSTEM == "local"
IS_HDFS = FILESYSTEM == "hdfs"
IS_ADLS = FILESYSTEM == "adls"
+IS_ABFS = FILESYSTEM == "abfs"
IS_EC = os.getenv("ERASURE_CODING") == "true"
# This condition satisfies both the states where one can assume a default fs
# - The environment variable is set to an empty string.
@@ -44,7 +45,9 @@ ISILON_WEBHDFS_PORT = 8082
# S3 specific values
S3_BUCKET_NAME = os.getenv("S3_BUCKET")
-# ADLS specific values
+# ADLS / ABFS specific values
+ABFS_ACCOUNT_NAME = os.getenv("azure_storage_account_name")
+ABFS_CONTAINER_NAME = os.getenv("azure_storage_container_name")
ADLS_STORE_NAME = os.getenv("azure_data_lake_store_name")
ADLS_CLIENT_ID = os.getenv("azure_client_id")
ADLS_TENANT_ID = os.getenv("azure_tenant_id")
[3/5] impala git commit: IMPALA-7699: Fix spilling test run with hdfs
erasure coding turned on
Posted by jo...@apache.org.
IMPALA-7699: Fix spilling test run with hdfs erasure coding turned on
A spilling test when run on test build with hdfs erasure coding turned
on hits an out of memory error on the hdfs scan node. This happens
because the test is tuned for a regular 3 node minicluster without
hdfs erasure coding. Fix is to simply increase the memory limit on
the test to accommodate this difference yet keep it small enough to
achieve desired spilling on the hash join node.
Testing:
Ran it on an EC enabled minicluster to make sure it works
Change-Id: I207569822ba7388e78936d25e2311fa09c7a1b9a
Reviewed-on: http://gerrit.cloudera.org:8080/11740
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/77c56a80
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/77c56a80
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/77c56a80
Branch: refs/heads/master
Commit: 77c56a805abf23db27db493ed12af965e515428d
Parents: bad49e7
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Fri Oct 19 13:28:20 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sat Oct 20 03:32:07 2018 +0000
----------------------------------------------------------------------
.../queries/QueryTest/spilling-no-debug-action.test | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/77c56a80/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
index 539a4fd..7fe1c96 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
@@ -91,7 +91,7 @@ Memory limit exceeded
# Hash join that will fail to repartition and therefore fail from out-of-memory because
# of a large number of duplicate keys on the build side: IMPALA-4857. It does not need
# any help from DEBUG_ACTION.
-set mem_limit=150m;
+set mem_limit=250m;
select straight_join *
from supplier join /* +broadcast */ lineitem on s_suppkey = l_linenumber
order by l_tax desc
[4/5] impala git commit: IMPALA-7697: Fix flakiness in
test_resource_limits
Posted by jo...@apache.org.
IMPALA-7697: Fix flakiness in test_resource_limits
This patch fixes one of the tests in test_resource_limits that expects a
query to run for more than 2 seconds but currently fails because it
sometimes completes earlier than that.
Change-Id: I2ba7080f62f0af3e16ef6c304463ebf78dec1b0c
Reviewed-on: http://gerrit.cloudera.org:8080/11741
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0340a153
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0340a153
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0340a153
Branch: refs/heads/master
Commit: 0340a153ceed2ac6897569faf158e357f8f628df
Parents: 77c56a8
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Fri Oct 19 15:02:13 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sat Oct 20 03:43:41 2018 +0000
----------------------------------------------------------------------
.../queries/QueryTest/query-resource-limits.test | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/0340a153/testdata/workloads/functional-query/queries/QueryTest/query-resource-limits.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/query-resource-limits.test b/testdata/workloads/functional-query/queries/QueryTest/query-resource-limits.test
index daba679..9301637 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/query-resource-limits.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/query-resource-limits.test
@@ -43,12 +43,7 @@ row_regex:.*terminated due to CPU limit of 1s000ms.*
---- QUERY
# Query should fail due to exceeding time limit.
set EXEC_TIME_LIMIT_S=2;
-set CPU_LIMIT_S=10000;
-set SCAN_BYTES_LIMIT="100G";
-select count(*) from tpch.lineitem l1,tpch.lineitem l2, tpch.lineitem l3 where
- l1.l_suppkey = l2.l_linenumber and l1.l_orderkey = l2.l_orderkey and
- l1.l_orderkey = l3.l_orderkey group by l1.l_comment, l2.l_comment
- having count(*) = 99
+select sleep(10000)
---- CATCH
row_regex:.*expired due to execution time limit of 2s000ms.*
====
[2/5] impala git commit: IMPALA-7721: Fix broken /catalog_object web
API when getting a privilege
Posted by jo...@apache.org.
IMPALA-7721: Fix broken /catalog_object web API when getting a privilege
Before this patch, /catalog_object web API was broken when getting a
privilege due to an incorrect way of getting a role ID. IMPALA-7616
broke this even more due to a lack of test coverage in /catalog_object
when authorization is enabled. This patch fixes the issue and makes the
/catalog_object web API usable again for getting a privilege.
Testing:
- Added a new BE test
- Added a new E2E test
- Ran all E2E authorization tests
Change-Id: I525149d113a1437c1e1493ad3c25a755e370b0c7
Reviewed-on: http://gerrit.cloudera.org:8080/11721
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bad49e73
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bad49e73
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bad49e73
Branch: refs/heads/master
Commit: bad49e73632f64a386ad1154201f99137af864d8
Parents: 072f3ee
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Wed Oct 17 16:02:49 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sat Oct 20 00:19:18 2018 +0000
----------------------------------------------------------------------
be/src/catalog/catalog-util-test.cc | 90 ++++++++++++-
be/src/catalog/catalog-util.cc | 125 ++++++++++++++++++-
be/src/catalog/catalog-util.h | 4 +-
.../java/org/apache/impala/catalog/Catalog.java | 12 +-
.../impala/catalog/PrincipalPrivilege.java | 2 +-
tests/authorization/test_authorization.py | 40 ++++++
6 files changed, 260 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/be/src/catalog/catalog-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog-util-test.cc b/be/src/catalog/catalog-util-test.cc
index d37fc5c..91466f5 100644
--- a/be/src/catalog/catalog-util-test.cc
+++ b/be/src/catalog/catalog-util-test.cc
@@ -15,11 +15,14 @@
// specific language governing permissions and limitations
// under the License.
+#include <gutil/strings/substitute.h>
+
#include "catalog/catalog-util.h"
#include "testutil/gtest-util.h"
using namespace impala;
using namespace std;
+using namespace strings;
void CompressAndDecompress(const std::string& input) {
string compressed;
@@ -32,7 +35,6 @@ void CompressAndDecompress(const std::string& input) {
ASSERT_EQ(input, decompressed);
}
-
TEST(CatalogUtil, TestCatalogCompression) {
CompressAndDecompress("");
CompressAndDecompress("deadbeef");
@@ -45,5 +47,91 @@ TEST(CatalogUtil, TestCatalogCompression) {
CompressAndDecompress(large_string);
}
+TEST(CatalogUtil, TestTPrivilegeFromObjectName) {
+ vector<tuple<string, TPrivilegeLevel::type>> actions = {
+ make_tuple("all", TPrivilegeLevel::ALL),
+ make_tuple("insert", TPrivilegeLevel::INSERT),
+ make_tuple("select", TPrivilegeLevel::SELECT),
+ make_tuple("refresh", TPrivilegeLevel::REFRESH),
+ make_tuple("create", TPrivilegeLevel::CREATE),
+ make_tuple("alter", TPrivilegeLevel::ALTER),
+ make_tuple("drop", TPrivilegeLevel::DROP),
+ make_tuple("owner", TPrivilegeLevel::OWNER)
+ };
+ vector<tuple<string, bool>> grant_options = {
+ make_tuple("true", true),
+ make_tuple("false", false)
+ };
+
+ for (const auto& action: actions) {
+ for (const auto& grant_option: grant_options) {
+ TPrivilege server_privilege;
+ ASSERT_OK(TPrivilegeFromObjectName(Substitute(
+ "server=server1->action=$0->grantoption=$1",
+ get<0>(action), get<0>(grant_option)), &server_privilege));
+ ASSERT_EQ(TPrivilegeScope::SERVER, server_privilege.scope);
+ ASSERT_EQ(get<1>(action), server_privilege.privilege_level);
+ ASSERT_EQ(get<1>(grant_option), server_privilege.has_grant_opt);
+ ASSERT_EQ("server1", server_privilege.server_name);
+
+ TPrivilege uri_privilege;
+ ASSERT_OK(TPrivilegeFromObjectName(Substitute(
+ "server=server1->uri=/test-warehouse->action=$0->grantoption=$1",
+ get<0>(action), get<0>(grant_option)), &uri_privilege));
+ ASSERT_EQ(TPrivilegeScope::URI, uri_privilege.scope);
+ ASSERT_EQ(get<1>(action), uri_privilege.privilege_level);
+ ASSERT_EQ(get<1>(grant_option), uri_privilege.has_grant_opt);
+ ASSERT_EQ("server1", uri_privilege.server_name);
+ ASSERT_EQ("/test-warehouse", uri_privilege.uri);
+
+ TPrivilege db_privilege;
+ ASSERT_OK(TPrivilegeFromObjectName(Substitute(
+ "server=server1->db=functional->action=$0->grantoption=$1",
+ get<0>(action), get<0>(grant_option)), &db_privilege));
+ ASSERT_EQ(TPrivilegeScope::DATABASE, db_privilege.scope);
+ ASSERT_EQ(get<1>(action), db_privilege.privilege_level);
+ ASSERT_EQ(get<1>(grant_option), db_privilege.has_grant_opt);
+ ASSERT_EQ("server1", db_privilege.server_name);
+ ASSERT_EQ("functional", db_privilege.db_name);
+
+ TPrivilege table_privilege;
+ ASSERT_OK(TPrivilegeFromObjectName(Substitute(
+ "server=server1->db=functional->table=alltypes->action=$0->grantoption=$1",
+ get<0>(action), get<0>(grant_option)), &table_privilege));
+ ASSERT_EQ(TPrivilegeScope::TABLE, table_privilege.scope);
+ ASSERT_EQ(get<1>(action), table_privilege.privilege_level);
+ ASSERT_EQ(get<1>(grant_option), table_privilege.has_grant_opt);
+ ASSERT_EQ("server1", table_privilege.server_name);
+ ASSERT_EQ("functional", table_privilege.db_name);
+ ASSERT_EQ("alltypes", table_privilege.table_name);
+
+ TPrivilege column_privilege;
+ ASSERT_OK(TPrivilegeFromObjectName(Substitute(
+ "server=server1->db=functional->table=alltypes->column=id->action=$0->"
+ "grantoption=$1", get<0>(action), get<0>(grant_option)), &column_privilege));
+ ASSERT_EQ(TPrivilegeScope::COLUMN, column_privilege.scope);
+ ASSERT_EQ(get<1>(action), column_privilege.privilege_level);
+ ASSERT_EQ(get<1>(grant_option), column_privilege.has_grant_opt);
+ ASSERT_EQ("server1", column_privilege.server_name);
+ ASSERT_EQ("functional", column_privilege.db_name);
+ ASSERT_EQ("alltypes", column_privilege.table_name);
+ ASSERT_EQ("id", column_privilege.column_name);
+ }
+ }
+
+ TPrivilege privilege;
+ EXPECT_ERROR(TPrivilegeFromObjectName("abc=server1->action=select->grantoption=true",
+ &privilege), TErrorCode::GENERAL);
+ EXPECT_ERROR(TPrivilegeFromObjectName("server=server1->action=foo->grantoption=true",
+ &privilege), TErrorCode::GENERAL);
+ EXPECT_ERROR(TPrivilegeFromObjectName("server=server1->action=select->grantoption=foo",
+ &privilege), TErrorCode::GENERAL);
+ EXPECT_ERROR(TPrivilegeFromObjectName("", &privilege), TErrorCode::GENERAL);
+ EXPECT_ERROR(TPrivilegeFromObjectName("SERVER=server1->action=select->grantoption=true",
+ &privilege), TErrorCode::GENERAL);
+ EXPECT_ERROR(TPrivilegeFromObjectName("server;server1->action=select->grantoption=true",
+ &privilege), TErrorCode::GENERAL);
+}
+
IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/be/src/catalog/catalog-util.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog-util.cc b/be/src/catalog/catalog-util.cc
index cc7bf49..5084828 100644
--- a/be/src/catalog/catalog-util.cc
+++ b/be/src/catalog/catalog-util.cc
@@ -17,6 +17,7 @@
#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string_regex.hpp>
#include <sstream>
#include "catalog/catalog-util.h"
@@ -24,6 +25,7 @@
#include "util/compress.h"
#include "util/jni-util.h"
#include "util/debug-util.h"
+#include "util/string-parser.h"
#include "common/names.h"
@@ -36,6 +38,10 @@ jmethodID JniCatalogCacheUpdateIterator::pair_ctor;
jclass JniCatalogCacheUpdateIterator::boolean_cl;
jmethodID JniCatalogCacheUpdateIterator::boolean_ctor;
+/// Populates a TPrivilegeLevel::type based on the given object name string.
+Status TPrivilegeLevelFromObjectName(const std::string& object_name,
+ TPrivilegeLevel::type* privilege_level);
+
Status JniCatalogCacheUpdateIterator::InitJNI() {
JNIEnv* env = getJNIEnv();
if (env == nullptr) return Status("Failed to get/create JVM");
@@ -202,16 +208,40 @@ Status TCatalogObjectFromObjectName(const TCatalogObjectType::type& object_type,
catalog_object->principal.__set_principal_name(object_name);
break;
case TCatalogObjectType::PRIVILEGE: {
- int pos = object_name.find(".");
- if (pos == string::npos || pos >= object_name.size() - 1) {
+ // The format is <privilege name>.<principal ID>.<principal type>
+ vector<string> split;
+ boost::split(split, object_name, [](char c){ return c == '.'; });
+ if (split.size() != 3) {
stringstream error_msg;
error_msg << "Invalid privilege name: " << object_name;
return Status(error_msg.str());
}
+ string privilege_name = split[0];
+ string principal_id = split[1];
+ string principal_type = split[2];
catalog_object->__set_type(object_type);
- catalog_object->__set_privilege(TPrivilege());
- catalog_object->privilege.__set_principal_id(
- atoi(object_name.substr(0, pos).c_str()));
+ TPrivilege privilege;
+ Status status = TPrivilegeFromObjectName(privilege_name, &privilege);
+ if (!status.ok()) return status;
+ catalog_object->__set_privilege(privilege);
+ StringParser::ParseResult result;
+ int32_t pid = StringParser::StringToInt<int32_t>(principal_id.c_str(),
+ principal_id.length(), &result);
+ if (result != StringParser::PARSE_SUCCESS) {
+ stringstream error_msg;
+ error_msg << "Invalid principal ID: " << principal_id;
+ return Status(error_msg.str());
+ }
+ catalog_object->privilege.__set_principal_id(pid);
+ if (principal_type == "ROLE") {
+ catalog_object->privilege.__set_principal_type(TPrincipalType::ROLE);
+ } else if (principal_type == "USER") {
+ catalog_object->privilege.__set_principal_type(TPrincipalType::USER);
+ } else {
+ stringstream error_msg;
+ error_msg << "Invalid principal type: " << principal_type;
+ return Status(error_msg.str());
+ }
break;
}
case TCatalogObjectType::CATALOG:
@@ -224,6 +254,64 @@ Status TCatalogObjectFromObjectName(const TCatalogObjectType::type& object_type,
return Status::OK();
}
+Status TPrivilegeFromObjectName(const string& object_name, TPrivilege* privilege) {
+ DCHECK(privilege != nullptr);
+ // Format:
+ // server=val->action=val->grantoption=[true|false]
+ // server=val->uri=val->action=val->grantoption=[true|false]
+ // server=val->db=val->action=val->grantoption=[true|false]
+ // server=val->db=val->table=val->action=val->grantoption=[true|false]
+ // server=val->db=val->table=val->column=val->action=val->grantoption=[true|false]
+ vector<string> split;
+ boost::algorithm::split_regex(split, object_name, boost::regex("->"));
+ for (const auto& s: split) {
+ vector<string> key_value;
+ boost::split(key_value, s, [](char c){ return c == '='; });
+ if (key_value.size() != 2) {
+ stringstream error_msg;
+ error_msg << "Invalid field name/value format: " << s;
+ return Status(error_msg.str());
+ }
+
+ if (key_value[0] == "server") {
+ privilege->__set_server_name(key_value[1]);
+ privilege->__set_scope(TPrivilegeScope::SERVER);
+ } else if (key_value[0] == "uri") {
+ privilege->__set_uri(key_value[1]);
+ privilege->__set_scope(TPrivilegeScope::URI);
+ } else if (key_value[0] == "db") {
+ privilege->__set_db_name(key_value[1]);
+ privilege->__set_scope(TPrivilegeScope::DATABASE);
+ } else if (key_value[0] == "table") {
+ privilege->__set_table_name(key_value[1]);
+ privilege->__set_scope(TPrivilegeScope::TABLE);
+ } else if (key_value[0] == "column") {
+ privilege->__set_column_name(key_value[1]);
+ privilege->__set_scope(TPrivilegeScope::COLUMN);
+ } else if (key_value[0] == "action") {
+ TPrivilegeLevel::type privilege_level;
+ Status status = TPrivilegeLevelFromObjectName(key_value[1], &privilege_level);
+ if (!status.ok()) return status;
+ privilege->__set_privilege_level(privilege_level);
+ } else if (key_value[0] == "grantoption") {
+ if (key_value[1] == "true") {
+ privilege->__set_has_grant_opt(true);
+ } else if (key_value[1] == "false") {
+ privilege->__set_has_grant_opt(false);
+ } else {
+ stringstream error_msg;
+ error_msg << "Invalid grant option: " << key_value[1];
+ return Status(error_msg.str());
+ }
+ } else {
+ stringstream error_msg;
+ error_msg << "Invalid privilege field name: " << key_value[0];
+ return Status(error_msg.str());
+ }
+ }
+ return Status::OK();
+}
+
Status CompressCatalogObject(const uint8_t* src, uint32_t size, string* dst) {
scoped_ptr<Codec> compressor;
RETURN_IF_ERROR(Codec::CreateCompressor(nullptr, false, THdfsCompression::LZ4,
@@ -252,4 +340,31 @@ Status DecompressCatalogObject(const uint8_t* src, uint32_t size, string* dst) {
return Status::OK();
}
+Status TPrivilegeLevelFromObjectName(const std::string& object_name,
+ TPrivilegeLevel::type* privilege_level) {
+ DCHECK(privilege_level != nullptr);
+ if (object_name == "all") {
+ *privilege_level = TPrivilegeLevel::ALL;
+ } else if (object_name == "insert") {
+ *privilege_level = TPrivilegeLevel::INSERT;
+ } else if (object_name == "select") {
+ *privilege_level = TPrivilegeLevel::SELECT;
+ } else if (object_name == "refresh") {
+ *privilege_level = TPrivilegeLevel::REFRESH;
+ } else if (object_name == "create") {
+ *privilege_level = TPrivilegeLevel::CREATE;
+ } else if (object_name == "alter") {
+ *privilege_level = TPrivilegeLevel::ALTER;
+ } else if (object_name == "drop") {
+ *privilege_level = TPrivilegeLevel::DROP;
+ } else if (object_name == "owner") {
+ *privilege_level = TPrivilegeLevel::OWNER;
+ } else {
+ stringstream error_msg;
+ error_msg << "Invalid privilege level: " << object_name;
+ return Status(error_msg.str());
+ }
+ return Status::OK();
+}
+
}
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/be/src/catalog/catalog-util.h
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog-util.h b/be/src/catalog/catalog-util.h
index a01e9bb..bc50408 100644
--- a/be/src/catalog/catalog-util.h
+++ b/be/src/catalog/catalog-util.h
@@ -98,6 +98,9 @@ TCatalogObjectType::type TCatalogObjectTypeFromName(const std::string& name);
Status TCatalogObjectFromObjectName(const TCatalogObjectType::type& object_type,
const std::string& object_name, TCatalogObject* catalog_object);
+/// Populates a TPrivilege based on the given object name string.
+Status TPrivilegeFromObjectName(const std::string& object_name, TPrivilege* privilege);
+
/// Compresses a serialized catalog object using LZ4 and stores it back in 'dst'. Stores
/// the size of the uncompressed catalog object in the first sizeof(uint32_t) bytes of
/// 'dst'. The compression fails if the uncompressed data size exceeds 0x7E000000 bytes.
@@ -109,7 +112,6 @@ Status CompressCatalogObject(const uint8_t* src, uint32_t size, std::string* dst
/// catalog object.
Status DecompressCatalogObject(const uint8_t* src, uint32_t size, std::string* dst)
WARN_UNUSED_RESULT;
-
}
#endif
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/fe/src/main/java/org/apache/impala/catalog/Catalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Catalog.java b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
index 879195f..fe49d5b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
@@ -512,11 +512,11 @@ public abstract class Catalog {
break;
case PRIVILEGE:
Principal tmpPrincipal = authPolicy_.getPrincipal(
- objectDesc.getPrincipal().getPrincipal_id(),
- objectDesc.getPrincipal().getPrincipal_type());
+ objectDesc.getPrivilege().getPrincipal_id(),
+ objectDesc.getPrivilege().getPrincipal_type());
if (tmpPrincipal == null) {
throw new CatalogException(String.format("No %s associated with ID: %d",
- Principal.toString(objectDesc.getPrincipal().getPrincipal_type())
+ Principal.toString(objectDesc.getPrivilege().getPrincipal_type())
.toLowerCase(), objectDesc.getPrivilege().getPrincipal_id()));
}
String privilegeName = PrincipalPrivilege.buildPrivilegeName(
@@ -561,11 +561,13 @@ public abstract class Catalog {
return "PRINCIPAL:" + catalogObject.getPrincipal().getPrincipal_name()
.toLowerCase();
case PRIVILEGE:
- // The combination of privilege name + principal ID is guaranteed to be unique.
+ // The combination of privilege name + principal ID + principal type is
+ // guaranteed to be unique.
return "PRIVILEGE:" +
PrincipalPrivilege.buildPrivilegeName(catalogObject.getPrivilege())
.toLowerCase() + "." +
- Integer.toString(catalogObject.getPrivilege().getPrincipal_id());
+ Integer.toString(catalogObject.getPrivilege().getPrincipal_id()) + "." +
+ catalogObject.getPrivilege().getPrincipal_type();
case HDFS_CACHE_POOL:
return "HDFS_CACHE_POOL:" +
catalogObject.getCache_pool().getPool_name().toLowerCase();
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java b/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
index 39046f0..c375cfc 100644
--- a/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
+++ b/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
@@ -146,7 +146,7 @@ public class PrincipalPrivilege extends CatalogObjectImpl {
@Override
public String getUniqueName() {
return "PRIVILEGE:" + getName().toLowerCase() + "." + Integer.toString(
- getPrincipalId());
+ getPrincipalId()) + "." + getPrincipalType().toString();
}
public TCatalogObject toTCatalogObject() {
http://git-wip-us.apache.org/repos/asf/impala/blob/bad49e73/tests/authorization/test_authorization.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index cf5b2e9..9da43d3 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -23,6 +23,9 @@ import shutil
import tempfile
import json
import grp
+import re
+import urllib
+
from time import sleep, time
from getpass import getuser
from ImpalaService import ImpalaHiveServer2Service
@@ -438,3 +441,40 @@ class TestAuthorization(CustomClusterTestSuite):
cols = row.split("\t")
return cols[0:len(cols) - 1]
assert map(columns, result.data) == expected
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--server_name=server1 --sentry_config=%s" % SENTRY_CONFIG_FILE,
+ catalogd_args="--sentry_config=%s" % SENTRY_CONFIG_FILE,
+ impala_log_dir=tempfile.mkdtemp(prefix="test_catalog_restart_",
+ dir=os.getenv("LOG_DIR")))
+ def test_catalog_object(self, unique_role):
+ """IMPALA-7721: Tests /catalog_object web API for principal and privilege"""
+ self.role_cleanup(unique_role)
+ try:
+ self.client.execute("create role %s" % unique_role)
+ self.client.execute("grant select on database functional to role %s" % unique_role)
+ for service in [self.cluster.catalogd.service,
+ self.cluster.get_first_impalad().service]:
+ obj_dump = service.get_catalog_object_dump("PRINCIPAL", unique_role)
+ assert "catalog_version" in obj_dump
+
+ # Get the privilege associated with that principal ID.
+ principal_id = re.search(r"principal_id \(i32\) = (\d+)", obj_dump)
+ assert principal_id is not None
+ obj_dump = service.get_catalog_object_dump("PRIVILEGE", urllib.quote(
+ "server=server1->db=functional->action=select->grantoption=false.%s.ROLE" %
+ principal_id.group(1)))
+ assert "catalog_version" in obj_dump
+
+ # Get the principal that does not exist.
+ obj_dump = service.get_catalog_object_dump("PRINCIPAL", "doesnotexist")
+ assert "CatalogException" in obj_dump
+
+ # Get the privilege that does not exist.
+ obj_dump = service.get_catalog_object_dump("PRIVILEGE", urllib.quote(
+ "server=server1->db=doesntexist->action=select->grantoption=false.%s.ROLE" %
+ principal_id.group(1)))
+ assert "CatalogException" in obj_dump
+ finally:
+ self.role_cleanup(unique_role)