You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/02/08 22:54:28 UTC
[impala] branch master updated (eca34cc98 -> 5abbb9bd1)
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from eca34cc98 IMPALA-11892: Restore pkg_resources with Python 2
new bbdc24d6f IMPALA-11834: Error reporting addendum
new 5abbb9bd1 IMPALA-11903: Ozone emits NONE when not erasure-coded
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/service/client-request-state.cc | 34 +++++++++++++++-------
.../org/apache/impala/common/FileSystemUtil.java | 6 ++++
2 files changed, 29 insertions(+), 11 deletions(-)
[impala] 02/02: IMPALA-11903: Ozone emits NONE when not erasure-coded
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5abbb9bd17373c8aafe6d213d328e16934cdca07
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Feb 7 13:05:57 2023 -0800
IMPALA-11903: Ozone emits NONE when not erasure-coded
Updates Ozone support for identifying erasure-coded files/tables to emit
NONE if not erasure-coded rather than the Ratis replication factor (e.g.
ONE, THREE, etc). Chose to do this for consistency as the output
specifically identifies the Erasure Coding Policy.
Testing:
- ran E2E tests with Ozone with and without EC
Change-Id: I1c3a34d4e108fed38b66f3dabefe867be5441b35
Reviewed-on: http://gerrit.cloudera.org:8080/19482
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/main/java/org/apache/impala/common/FileSystemUtil.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index 5ed8b4cae..e50fb85ba 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -241,6 +241,12 @@ public class FileSystemUtil {
} else if (isOzoneFileSystem(p)) {
try {
FileSystem fs = p.getFileSystem(CONF);
+ if (!fs.getFileStatus(p).isErasureCoded()) {
+ // Ozone will return the replication factor (ONE, THREE, etc) for Ratis
+ // replication. We avoid returning that for consistency.
+ return NO_ERASURE_CODE_LABEL;
+ }
+
if (fs instanceof BasicRootedOzoneFileSystem) {
BasicRootedOzoneFileSystem ofs = (BasicRootedOzoneFileSystem) fs;
Preconditions.checkState(
[impala] 01/02: IMPALA-11834: Error reporting addendum
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit bbdc24d6fc40c0f57f75ced543d8331fb0f33fd9
Author: Tamas Mate <tm...@apache.org>
AuthorDate: Fri Feb 3 11:56:24 2023 +0100
IMPALA-11834: Error reporting addendum
This commit is an addendum that addresses some review comments that
arrived while the base change was merged. Code cleanup and improved
error reporting.
Testing:
- Tested the code paths manually
Change-Id: Iece31fa19b7448095462671acb2f02e8c5a405ae
Reviewed-on: http://gerrit.cloudera.org:8080/19481
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/client-request-state.cc | 34 +++++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index 13c9bef63..e0c066de6 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -872,28 +872,40 @@ void ClientRequestState::ExecLoadIcebergDataRequestImpl(TLoadDataResp response)
Status query_status = child_query_executor_->WaitForAll(completed_queries);
if (query_status.ok()) {
const char* path = response.create_location.c_str();
+ string delete_err = "Load was succesful, but failed to remove staging data under '"
+ + response.create_location + "', HDFS error: ";
hdfsFS hdfs_conn;
Status hdfs_ret = HdfsFsCache::instance()->GetConnection(path, &hdfs_conn);
- if (!hdfs_ret.ok() || hdfsDelete(hdfs_conn, path, 1)) {
- hdfs_ret = Status("Failed to remove staging data under '" + response.create_location
- + "' after query failure: " + query_status.msg().msg());
+ if (!hdfs_ret.ok()) {
lock_guard<mutex> l(lock_);
- RETURN_VOID_IF_ERROR(UpdateQueryStatus(hdfs_ret));
+ RETURN_VOID_IF_ERROR(UpdateQueryStatus(Status(delete_err + hdfs_ret.GetDetail())));
+ }
+ if (hdfsDelete(hdfs_conn, path, 1)) {
+ lock_guard<mutex> l(lock_);
+ RETURN_VOID_IF_ERROR(UpdateQueryStatus(Status(delete_err + strerror(errno))));
}
} else {
const char* dst_path = load_data_req.source_path.c_str();
hdfsFS hdfs_dst_conn;
+ string revert_err = "Failed to load data and failed to revert data movement, "
+ "please check source and staging directory under '" + response.create_location
+ + "', Query error: " + query_status.GetDetail() + " HDFS error: ";
Status hdfs_ret = HdfsFsCache::instance()->GetConnection(dst_path, &hdfs_dst_conn);
+ if (!hdfs_ret.ok()) {
+ lock_guard<mutex> l(lock_);
+ RETURN_VOID_IF_ERROR(UpdateQueryStatus(Status(revert_err + hdfs_ret.GetDetail())));
+ }
for (string src_path : response.loaded_files) {
hdfsFS hdfs_src_conn;
- hdfs_ret = Status(HdfsFsCache::instance()->GetConnection(dst_path, &hdfs_src_conn));
- if (!hdfs_ret.ok() || hdfsMove(hdfs_src_conn, src_path.c_str(), hdfs_dst_conn,
- dst_path)) {
- hdfs_ret = Status("Failed to revert data movement, some files might still be in '"
- + response.create_location + "' after query failure: "
- + query_status.msg().msg());
+ hdfs_ret = HdfsFsCache::instance()->GetConnection(dst_path, &hdfs_src_conn);
+ if (!hdfs_ret.ok()) {
+ lock_guard<mutex> l(lock_);
+ RETURN_VOID_IF_ERROR(UpdateQueryStatus(Status(revert_err
+ + hdfs_ret.GetDetail())));
+ }
+ if (hdfsMove(hdfs_src_conn, src_path.c_str(), hdfs_dst_conn, dst_path)) {
lock_guard<mutex> l(lock_);
- RETURN_VOID_IF_ERROR(UpdateQueryStatus(hdfs_ret));
+ RETURN_VOID_IF_ERROR(UpdateQueryStatus(Status(revert_err + strerror(errno))));
}
}
}