You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/09/12 07:16:14 UTC
[3/4] incubator-impala git commit: IMPALA-3642: Adding backend
addresses to error statuses for some scratch failures.
IMPALA-3642: Adding backend addresses to error statuses for some scratch failures.
Adds GetBackendAddress() (which is host:port) to error messages stemming
from SCRATCH_LIMIT_EXCEEDED, SCRATCH_READ_TRUNCATED, and
SCRATCH_ALLOCATION_FAILED messages.
Testing:
* Unit tests assert the string is updated for SCRATCH_LIMIT_EXCEEDED
and SCRATCH_ALLOCATION_FAILED. SCRATCH_READ_TRUNCATED doesn't
have an existing test, and I didn't add a new one.
* Manually testing a query that spills after "chmod 000 /tmp/impala-scratch":
$ chmod 000 /tmp/impala-scratch
$ impala-shell
[dev:21000] > set mem_limit=100m;
MEM_LIMIT set to 100m
[dev:21000] > select count(*) from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey;
Query: select count(*) from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
Query submitted at: 2017-09-11 11:07:06 (Coordinator: http://dev:25000)
Query progress can be monitored at: http://dev:25000/query_plan?query_id=5c48ff8f4103c194:1b40a6c00000000
WARNINGS: Could not create files in any configured scratch directories (--scratch_dirs=/tmp/impala-scratch) on backend 'dev:22002'. See logs for previous errors that may have prevented creating or writing scratch files.
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Change-Id: If31a50fdf6031312d0348d48aeb8f9688274cac2
Reviewed-on: http://gerrit.cloudera.org:8080/7816
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/39f23bb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/39f23bb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/39f23bb8
Branch: refs/heads/master
Commit: 39f23bb8b9d7cd01c3e31962c9b9e35c4827a593
Parents: 4e84262
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Thu Aug 24 08:45:08 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Sep 12 03:54:21 2017 +0000
----------------------------------------------------------------------
be/src/runtime/bufferpool/buffer-pool-test.cc | 1 +
be/src/runtime/tmp-file-mgr-test.cc | 1 +
be/src/runtime/tmp-file-mgr.cc | 10 ++++++----
common/thrift/generate_error_codes.py | 10 +++++-----
4 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/bufferpool/buffer-pool-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-pool-test.cc b/be/src/runtime/bufferpool/buffer-pool-test.cc
index 61d7a39..06ff827 100644
--- a/be/src/runtime/bufferpool/buffer-pool-test.cc
+++ b/be/src/runtime/bufferpool/buffer-pool-test.cc
@@ -1460,6 +1460,7 @@ void BufferPoolTest::TestWriteError(int write_delay_ms) {
PageHandle tmp_page;
Status error = pool.AllocateBuffer(&client, TEST_BUFFER_LEN, &tmp_buffer);
EXPECT_EQ(TErrorCode::SCRATCH_ALLOCATION_FAILED, error.code());
+ ASSERT_NE(string::npos, error.msg().msg().find(GetBackendString()));
EXPECT_FALSE(tmp_buffer.is_open());
error = pool.CreatePage(&client, TEST_BUFFER_LEN, &tmp_page);
EXPECT_EQ(TErrorCode::SCRATCH_ALLOCATION_FAILED, error.code());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/tmp-file-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr-test.cc b/be/src/runtime/tmp-file-mgr-test.cc
index c610529..1a5eb58 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -384,6 +384,7 @@ TEST_F(TmpFileMgrTest, TestScratchLimit) {
status = GroupAllocateSpace(&file_group, 1, &alloc_file, &offset);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), TErrorCode::SCRATCH_LIMIT_EXCEEDED);
+ ASSERT_NE(string::npos, status.msg().msg().find(GetBackendString()));
file_group.Close();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/tmp-file-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index 1cf86b4..8916d4f 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -269,7 +269,8 @@ Status TmpFileMgr::FileGroup::CreateFiles() {
}
DCHECK_EQ(tmp_files_.size(), files_allocated);
if (tmp_files_.size() == 0) {
- Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED);
+ Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED,
+ join(tmp_file_mgr_->tmp_dirs_, ","), GetBackendString());
for (Status& err : scratch_errors_) err_status.MergeStatus(err);
return err_status;
}
@@ -307,7 +308,7 @@ Status TmpFileMgr::FileGroup::AllocateSpace(
if (bytes_limit_ != -1
&& current_bytes_allocated_ + scratch_range_bytes > bytes_limit_) {
- return Status(TErrorCode::SCRATCH_LIMIT_EXCEEDED, bytes_limit_);
+ return Status(TErrorCode::SCRATCH_LIMIT_EXCEEDED, bytes_limit_, GetBackendString());
}
// Lazily create the files on the first write.
@@ -331,7 +332,8 @@ Status TmpFileMgr::FileGroup::AllocateSpace(
<< ". Will try another scratch file.";
scratch_errors_.push_back(status);
}
- Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED);
+ Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED,
+ join(tmp_file_mgr_->tmp_dirs_, ","), GetBackendString());
// Include all previous errors that may have caused the failure.
for (Status& err : scratch_errors_) err_status.MergeStatus(err);
return err_status;
@@ -410,7 +412,7 @@ Status TmpFileMgr::FileGroup::WaitForAsyncRead(WriteHandle* handle, MemRange buf
if (io_mgr_buffer->len() < buffer.len()) {
// The read was truncated - this is an error.
status = Status(TErrorCode::SCRATCH_READ_TRUNCATED, buffer.len(),
- handle->write_range_->file(), handle->write_range_->offset(),
+ handle->write_range_->file(), GetBackendString(), handle->write_range_->offset(),
io_mgr_buffer->len());
goto exit;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/common/thrift/generate_error_codes.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index c08268e..3c37bef 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -275,7 +275,7 @@ error_codes = (
"supported length of 2147483647 bytes."),
("SCRATCH_LIMIT_EXCEEDED", 90, "Scratch space limit of $0 bytes exceeded for query "
- "while spilling data to disk."),
+ "while spilling data to disk on backend $1."),
("BUFFER_ALLOCATION_FAILED", 91, "Unexpected error allocating $0 byte buffer: $1"),
@@ -306,11 +306,11 @@ error_codes = (
# TODO: IMPALA-4697: the merged errors do not show up in the query error log,
# so we must point users to the impalad error log.
("SCRATCH_ALLOCATION_FAILED", 101, "Could not create files in any configured scratch "
- "directories (--scratch_dirs). See logs for previous errors that may have prevented "
- "creating or writing scratch files."),
+ "directories (--scratch_dirs=$0) on backend '$1'. See logs for previous errors that may "
+ "have prevented creating or writing scratch files."),
- ("SCRATCH_READ_TRUNCATED", 102, "Error reading $0 bytes from scratch file '$1' at "
- "offset $2: could only read $3 bytes"),
+ ("SCRATCH_READ_TRUNCATED", 102, "Error reading $0 bytes from scratch file '$1' "
+ "on backend $2 at offset $3: could only read $4 bytes"),
("KUDU_TIMESTAMP_OUT_OF_RANGE", 103,
"Kudu table '$0' column '$1' contains an out of range timestamp. "