You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/09/12 07:16:12 UTC
[1/4] incubator-impala git commit: IMPALA-5912: fix crash in
trunc(..., "WW") in release build
Repository: incubator-impala
Updated Branches:
refs/heads/master 2fbdc8e37 -> 03571788b
IMPALA-5912: fix crash in trunc(..., "WW") in release build
The bug is with the pattern below:
const date& d = TruncMonth(orig_date).date();
The problem is that 'd' is a reference into the temporary returned from
TruncMonth. But the temporary is only guaranteed to live until the
expression has been evaluated. Thus if the compiler re-uses the register
or stack slot holding the temporary, 'd' may end up pointing to a bogus
value, causing a crash or incorrect results.
The fix is to simply create a local date value with the required date,
which avoids use of references to expression temporary and makes the
logic more obviously correct.
Also remove other uses of references to temporary that were correct but
unnecessary given that the function returned a value and C++11
return value optimisation should kick in.
Testing:
Ran expr-test to completion with a release build. Before this fix it
reliably crashed for me.
Change-Id: Ic5017518188f5025daa5040ca1943581a0675726
Reviewed-on: http://gerrit.cloudera.org:8080/8015
Reviewed-by: Thomas Tauber-Marshall <tm...@cloudera.com>
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e1ae9884
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e1ae9884
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e1ae9884
Branch: refs/heads/master
Commit: e1ae9884168b0c455fa147bb10dc57d37989f9e8
Parents: 2fbdc8e
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Sep 8 15:14:00 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Sep 12 01:53:20 2017 +0000
----------------------------------------------------------------------
be/src/exprs/udf-builtins.cc | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e1ae9884/be/src/exprs/udf-builtins.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/udf-builtins.cc b/be/src/exprs/udf-builtins.cc
index 01178e3..ebfca35 100644
--- a/be/src/exprs/udf-builtins.cc
+++ b/be/src/exprs/udf-builtins.cc
@@ -193,7 +193,7 @@ TimestampValue TruncWeek(const date& orig_date) {
// Same day of the week as the first day of the year
TimestampValue TruncWW(const date& orig_date) {
- const date& first_day_of_year = TruncYear(orig_date).date();
+ date first_day_of_year(orig_date.year(), 1, 1);
int target_week_day = first_day_of_year.day_of_week();
date new_date = GoBackToWeekday(orig_date, target_week_day);
time_duration new_time(0, 0, 0, 0);
@@ -202,8 +202,8 @@ TimestampValue TruncWW(const date& orig_date) {
// Same day of the week as the first day of the month
TimestampValue TruncW(const date& orig_date) {
- const date& first_day_of_mon = TruncMonth(orig_date).date();
- const date& new_date = GoBackToWeekday(orig_date, first_day_of_mon.day_of_week());
+ date first_day_of_mon(orig_date.year(), orig_date.month(), 1);
+ date new_date = GoBackToWeekday(orig_date, first_day_of_mon.day_of_week());
time_duration new_time(0, 0, 0, 0);
return TimestampValue(new_date, new_time);
}
@@ -216,7 +216,7 @@ TimestampValue TruncDay(const date& orig_date) {
// Date of the previous Monday
TimestampValue TruncDayOfWeek(const date& orig_date) {
- const date& new_date = GoBackToWeekday(orig_date, 1);
+ date new_date = GoBackToWeekday(orig_date, 1);
time_duration new_time(0, 0, 0, 0);
return TimestampValue(new_date, new_time);
}
@@ -364,7 +364,7 @@ TimestampVal DoTrunc(
TimestampVal UdfBuiltins::TruncImpl(
FunctionContext* context, const TimestampVal& tv, const StringVal& unit_str) {
if (tv.is_null) return TimestampVal::null();
- const TimestampValue& ts = TimestampValue::FromTimestampVal(tv);
+ TimestampValue ts = TimestampValue::FromTimestampVal(tv);
// resolve trunc_unit using the prepared state if possible, o.w. parse now
// TruncPrepare() can only parse trunc_unit if user passes it as a string literal
@@ -413,7 +413,7 @@ void UdfBuiltins::TruncClose(FunctionContext* ctx,
TimestampVal UdfBuiltins::DateTruncImpl(
FunctionContext* context, const TimestampVal& tv, const StringVal& unit_str) {
if (tv.is_null) return TimestampVal::null();
- const TimestampValue& ts = TimestampValue::FromTimestampVal(tv);
+ TimestampValue ts = TimestampValue::FromTimestampVal(tv);
// resolve date_trunc_unit using the prepared state if possible, o.w. parse now
// DateTruncPrepare() can only parse trunc_unit if user passes it as a string literal
[2/4] incubator-impala git commit: IMPALA-3897 Codegen null-aware
constant in PHJ::ProcessBuildBatch()
Posted by jr...@apache.org.
IMPALA-3897 Codegen null-aware constant in PHJ::ProcessBuildBatch()
This change codegen outs a branch in ProcessBuildBatch(). This branch never gets
executed for most of the join types except NULL_AWARE_LEFT_ANTI_JOIN.
The branch itself is not expensive to execute, but it will reduce codegen
time by removing the dead code inside the branch for almost all join
modes.
Change-Id: I06acbebc9d2d23bef4734b480a5d3ce41680ea70
Reviewed-on: http://gerrit.cloudera.org:8080/7849
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4e842620
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4e842620
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4e842620
Branch: refs/heads/master
Commit: 4e84262074cab9cc8ad83a5a9d5bc42b4be42d76
Parents: e1ae988
Author: aphadke <ap...@cloudera.com>
Authored: Fri Aug 18 17:02:02 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Sep 12 03:46:21 2017 +0000
----------------------------------------------------------------------
be/src/codegen/gen_ir_descriptions.py | 2 +-
be/src/exec/partitioned-hash-join-builder-ir.cc | 5 ++---
be/src/exec/partitioned-hash-join-builder.cc | 15 ++++++++++++---
be/src/exec/partitioned-hash-join-builder.h | 8 +++++---
4 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e842620/be/src/codegen/gen_ir_descriptions.py
----------------------------------------------------------------------
diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 668ae24..75d233c 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -99,7 +99,7 @@ ir_functions = [
["HASH_FNV", "IrFnvHash"],
["HASH_MURMUR", "IrMurmurHash"],
["PHJ_PROCESS_BUILD_BATCH",
- "_ZN6impala10PhjBuilder17ProcessBuildBatchEPNS_8RowBatchEPNS_12HashTableCtxEb"],
+ "_ZN6impala10PhjBuilder17ProcessBuildBatchEPNS_8RowBatchEPNS_12HashTableCtxEbb"],
["PHJ_PROCESS_PROBE_BATCH_INNER_JOIN",
"_ZN6impala23PartitionedHashJoinNode17ProcessProbeBatchILi0EEEiNS_13TPrefetchMode4typeEPNS_8RowBatchEPNS_12HashTableCtxEPNS_6StatusE"],
["PHJ_PROCESS_PROBE_BATCH_LEFT_OUTER_JOIN",
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e842620/be/src/exec/partitioned-hash-join-builder-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-hash-join-builder-ir.cc b/be/src/exec/partitioned-hash-join-builder-ir.cc
index e15e116..b9c2cc3 100644
--- a/be/src/exec/partitioned-hash-join-builder-ir.cc
+++ b/be/src/exec/partitioned-hash-join-builder-ir.cc
@@ -37,15 +37,14 @@ inline bool PhjBuilder::AppendRow(
}
Status PhjBuilder::ProcessBuildBatch(
- RowBatch* build_batch, HashTableCtx* ctx, bool build_filters) {
+ RowBatch* build_batch, HashTableCtx* ctx, bool build_filters, bool is_null_aware) {
Status status;
HashTableCtx::ExprValuesCache* expr_vals_cache = ctx->expr_values_cache();
expr_vals_cache->Reset();
FOREACH_ROW(build_batch, 0, build_batch_iter) {
TupleRow* build_row = build_batch_iter.Get();
if (!ctx->EvalAndHashBuild(build_row)) {
- if (null_aware_partition_ != NULL) {
- // TODO: remove with codegen/template
+ if (is_null_aware) {
// If we are NULL aware and this build row has NULL in the eq join slot,
// append it to the null_aware partition. We will need it later.
if (UNLIKELY(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e842620/be/src/exec/partitioned-hash-join-builder.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-hash-join-builder.cc b/be/src/exec/partitioned-hash-join-builder.cc
index 0c04541..d292c59 100644
--- a/be/src/exec/partitioned-hash-join-builder.cc
+++ b/be/src/exec/partitioned-hash-join-builder.cc
@@ -191,14 +191,18 @@ Status PhjBuilder::Send(RuntimeState* state, RowBatch* batch) {
SCOPED_TIMER(partition_build_rows_timer_);
bool build_filters = ht_ctx_->level() == 0 && filter_ctxs_.size() > 0;
if (process_build_batch_fn_ == NULL) {
- RETURN_IF_ERROR(ProcessBuildBatch(batch, ht_ctx_.get(), build_filters));
+ RETURN_IF_ERROR(ProcessBuildBatch(batch, ht_ctx_.get(), build_filters,
+ join_op_ == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN));
+
} else {
DCHECK(process_build_batch_fn_level0_ != NULL);
if (ht_ctx_->level() == 0) {
RETURN_IF_ERROR(
- process_build_batch_fn_level0_(this, batch, ht_ctx_.get(), build_filters));
+ process_build_batch_fn_level0_(this, batch, ht_ctx_.get(), build_filters,
+ join_op_ == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN));
} else {
- RETURN_IF_ERROR(process_build_batch_fn_(this, batch, ht_ctx_.get(), build_filters));
+ RETURN_IF_ERROR(process_build_batch_fn_(this, batch, ht_ctx_.get(), build_filters,
+ join_op_ == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN));
}
}
@@ -807,6 +811,11 @@ Status PhjBuilder::CodegenProcessBuildBatch(LlvmCodeGen* codegen,
DCHECK_EQ(replaced_constants.stores_tuples, 0);
DCHECK_EQ(replaced_constants.quadratic_probing, 0);
+ Value* is_null_aware_arg = codegen->GetArgument(process_build_batch_fn, 5);
+ is_null_aware_arg->replaceAllUsesWith(
+ ConstantInt::get(Type::getInt1Ty(codegen->context()),
+ join_op_ == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN));
+
Function* process_build_batch_fn_level0 =
codegen->CloneFunction(process_build_batch_fn);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e842620/be/src/exec/partitioned-hash-join-builder.h
----------------------------------------------------------------------
diff --git a/be/src/exec/partitioned-hash-join-builder.h b/be/src/exec/partitioned-hash-join-builder.h
index eba9e9f..2c52988 100644
--- a/be/src/exec/partitioned-hash-join-builder.h
+++ b/be/src/exec/partitioned-hash-join-builder.h
@@ -290,9 +290,11 @@ class PhjBuilder : public DataSink {
Status CreateAndPreparePartition(int level, Partition** partition) WARN_UNUSED_RESULT;
/// Reads the rows in build_batch and partitions them into hash_partitions_. If
- /// 'build_filters' is true, runtime filters are populated.
+ /// 'build_filters' is true, runtime filters are populated. 'is_null_aware' is
+ /// set to true if the join type is a null aware join.
Status ProcessBuildBatch(
- RowBatch* build_batch, HashTableCtx* ctx, bool build_filters) WARN_UNUSED_RESULT;
+ RowBatch* build_batch, HashTableCtx* ctx, bool build_filters,
+ bool is_null_aware) WARN_UNUSED_RESULT;
/// Append 'row' to 'stream'. In the common case, appending the row to the stream
/// immediately succeeds. Otherwise this function falls back to the slower path of
@@ -491,7 +493,7 @@ class PhjBuilder : public DataSink {
/// and is used when the partition level is 0, otherwise xxx_fn_ uses murmur hash and is
/// used for subsequent levels.
typedef Status (*ProcessBuildBatchFn)(
- PhjBuilder*, RowBatch*, HashTableCtx*, bool build_filters);
+ PhjBuilder*, RowBatch*, HashTableCtx*, bool build_filters, bool is_null_aware);
/// Jitted ProcessBuildBatch function pointers. NULL if codegen is disabled.
ProcessBuildBatchFn process_build_batch_fn_;
ProcessBuildBatchFn process_build_batch_fn_level0_;
[4/4] incubator-impala git commit: IMPALA-5894: [DOCS] Clarify
placement of STRAIGHT_JOIN hint
Posted by jr...@apache.org.
IMPALA-5894: [DOCS] Clarify placement of STRAIGHT_JOIN hint
"immediately after the SELECT keyword" was mentioned in a
few places for STRAIGHT_JOIN. I reworded all instances to
mention that [DISTINCT | ALL] can also come before the
hint name.
Change-Id: I3cac1afccc132f389b2017ad217fdf7e7b04513a
Reviewed-on: http://gerrit.cloudera.org:8080/8031
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/03571788
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/03571788
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/03571788
Branch: refs/heads/master
Commit: 03571788be44d903a81dc092a447c74ccb9bac76
Parents: 39f23bb
Author: John Russell <jr...@cloudera.com>
Authored: Mon Sep 11 12:05:41 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Sep 12 07:02:09 2017 +0000
----------------------------------------------------------------------
docs/shared/impala_common.xml | 2 +-
docs/topics/impala_hints.xml | 3 ++-
docs/topics/impala_perf_joins.xml | 12 +++++++++---
3 files changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/03571788/docs/shared/impala_common.xml
----------------------------------------------------------------------
diff --git a/docs/shared/impala_common.xml b/docs/shared/impala_common.xml
index 9d6f72b..f31bdf0 100644
--- a/docs/shared/impala_common.xml
+++ b/docs/shared/impala_common.xml
@@ -2722,7 +2722,7 @@ flight_num: INT32 SNAPPY DO:83456393 FPO:83488603 SZ:10216514/11474301
are supported in all Impala versions. The <codeph>CROSS JOIN</codeph> operator is available in Impala 1.2.2
and higher. During performance tuning, you can override the reordering of join clauses that Impala does
internally by including the keyword <codeph>STRAIGHT_JOIN</codeph> immediately after the
- <codeph>SELECT</codeph> keyword
+ <codeph>SELECT</codeph> and any <codeph>DISTINCT</codeph> or <codeph>ALL</codeph> keywords.
</p>
<p id="catalog_server_124">
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/03571788/docs/topics/impala_hints.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_hints.xml b/docs/topics/impala_hints.xml
index 18b0dc6..843d9be 100644
--- a/docs/topics/impala_hints.xml
+++ b/docs/topics/impala_hints.xml
@@ -131,7 +131,8 @@ INSERT <varname>insert_clauses</varname>
<p>
With both forms of hint syntax, include the <codeph>STRAIGHT_JOIN</codeph>
- keyword immediately after the <codeph>SELECT</codeph> keyword to prevent Impala from
+ keyword immediately after the <codeph>SELECT</codeph> and any
+ <codeph>DISTINCT</codeph> or <codeph>ALL</codeph> keywords to prevent Impala from
reordering the tables in a way that makes the join-related hints ineffective.
</p>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/03571788/docs/topics/impala_perf_joins.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_perf_joins.xml b/docs/topics/impala_perf_joins.xml
index 116ba75..9cae4ef 100644
--- a/docs/topics/impala_perf_joins.xml
+++ b/docs/topics/impala_perf_joins.xml
@@ -57,8 +57,9 @@ under the License.
<p>
If statistics are not available for all the tables in the join query, or if Impala chooses a join order that
is not the most efficient, you can override the automatic join order optimization by specifying the
- <codeph>STRAIGHT_JOIN</codeph> keyword immediately after the <codeph>SELECT</codeph> keyword. In this case,
- Impala uses the order the tables appear in the query to guide how the joins are processed.
+ <codeph>STRAIGHT_JOIN</codeph> keyword immediately after the <codeph>SELECT</codeph> and any <codeph>DISTINCT</codeph>
+ or <codeph>ALL</codeph> keywords. In this case, Impala uses the order the tables appear in the query to guide how the
+ joins are processed.
</p>
<p>
@@ -149,7 +150,8 @@ under the License.
<p>
If an Impala join query is inefficient because of outdated statistics or unexpected data distribution, you
can keep Impala from reordering the joined tables by using the <codeph>STRAIGHT_JOIN</codeph> keyword
- immediately after the <codeph>SELECT</codeph> keyword. The <codeph>STRAIGHT_JOIN</codeph> keyword turns off
+ immediately after the <codeph>SELECT</codeph> and any <codeph>DISTINCT</codeph> or <codeph>ALL</codeph>
+ keywords. The <codeph>STRAIGHT_JOIN</codeph> keyword turns off
the reordering of join clauses that Impala does internally, and produces a plan that relies on the join
clauses being ordered optimally in the query text. In this case, rewrite the query so that the largest
table is on the left, followed by the next largest, and so on until the smallest table is on the right.
@@ -163,6 +165,10 @@ under the License.
</p>
<codeblock>select straight_join x from medium join small join (select * from big where c1 < 10) as big
+ where medium.id = small.id and small.id = big.id;
+
+-- If the query contains [DISTINCT | ALL], the hint goes after those keywords.
+select distinct straight_join x from medium join small join (select * from big where c1 < 10) as big
where medium.id = small.id and small.id = big.id;</codeblock>
</conbody>
</concept>
[3/4] incubator-impala git commit: IMPALA-3642: Adding backend
addresses to error statuses for some scratch failures.
Posted by jr...@apache.org.
IMPALA-3642: Adding backend addresses to error statuses for some scratch failures.
Adds GetBackendAddress() (which is host:port) to error messages stemming
from SCRATCH_LIMIT_EXCEEDED, SCRATCH_READ_TRUNCATED, and
SCRATCH_ALLOCATION_FAILED messages.
Testing:
* Unit tests assert the string is updated for SCRATCH_LIMIT_EXCEEDED
and SCRATCH_ALLOCATION_FAILED. SCRATCH_READ_TRUNCATED doesn't
have an existing test, and I didn't add a new one.
* Manually testing a query that spills after "chmod 000 /tmp/impala-scratch":
$ chmod 000 /tmp/impala-scratch
$ impala-shell
[dev:21000] > set mem_limit=100m;
MEM_LIMIT set to 100m
[dev:21000] > select count(*) from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey;
Query: select count(*) from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
Query submitted at: 2017-09-11 11:07:06 (Coordinator: http://dev:25000)
Query progress can be monitored at: http://dev:25000/query_plan?query_id=5c48ff8f4103c194:1b40a6c00000000
WARNINGS: Could not create files in any configured scratch directories (--scratch_dirs=/tmp/impala-scratch) on backend 'dev:22002'. See logs for previous errors that may have prevented creating or writing scratch files.
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Opening '/tmp/impala-scratch/5c48ff8f4103c194:1b40a6c00000000_08e8d63b-169d-4571-a0fe-c48fa08d73e6' for write failed with errno=13 description=Error(13): Permission denied
Change-Id: If31a50fdf6031312d0348d48aeb8f9688274cac2
Reviewed-on: http://gerrit.cloudera.org:8080/7816
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/39f23bb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/39f23bb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/39f23bb8
Branch: refs/heads/master
Commit: 39f23bb8b9d7cd01c3e31962c9b9e35c4827a593
Parents: 4e84262
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Thu Aug 24 08:45:08 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Sep 12 03:54:21 2017 +0000
----------------------------------------------------------------------
be/src/runtime/bufferpool/buffer-pool-test.cc | 1 +
be/src/runtime/tmp-file-mgr-test.cc | 1 +
be/src/runtime/tmp-file-mgr.cc | 10 ++++++----
common/thrift/generate_error_codes.py | 10 +++++-----
4 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/bufferpool/buffer-pool-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-pool-test.cc b/be/src/runtime/bufferpool/buffer-pool-test.cc
index 61d7a39..06ff827 100644
--- a/be/src/runtime/bufferpool/buffer-pool-test.cc
+++ b/be/src/runtime/bufferpool/buffer-pool-test.cc
@@ -1460,6 +1460,7 @@ void BufferPoolTest::TestWriteError(int write_delay_ms) {
PageHandle tmp_page;
Status error = pool.AllocateBuffer(&client, TEST_BUFFER_LEN, &tmp_buffer);
EXPECT_EQ(TErrorCode::SCRATCH_ALLOCATION_FAILED, error.code());
+ ASSERT_NE(string::npos, error.msg().msg().find(GetBackendString()));
EXPECT_FALSE(tmp_buffer.is_open());
error = pool.CreatePage(&client, TEST_BUFFER_LEN, &tmp_page);
EXPECT_EQ(TErrorCode::SCRATCH_ALLOCATION_FAILED, error.code());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/tmp-file-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr-test.cc b/be/src/runtime/tmp-file-mgr-test.cc
index c610529..1a5eb58 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -384,6 +384,7 @@ TEST_F(TmpFileMgrTest, TestScratchLimit) {
status = GroupAllocateSpace(&file_group, 1, &alloc_file, &offset);
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), TErrorCode::SCRATCH_LIMIT_EXCEEDED);
+ ASSERT_NE(string::npos, status.msg().msg().find(GetBackendString()));
file_group.Close();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/be/src/runtime/tmp-file-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index 1cf86b4..8916d4f 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -269,7 +269,8 @@ Status TmpFileMgr::FileGroup::CreateFiles() {
}
DCHECK_EQ(tmp_files_.size(), files_allocated);
if (tmp_files_.size() == 0) {
- Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED);
+ Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED,
+ join(tmp_file_mgr_->tmp_dirs_, ","), GetBackendString());
for (Status& err : scratch_errors_) err_status.MergeStatus(err);
return err_status;
}
@@ -307,7 +308,7 @@ Status TmpFileMgr::FileGroup::AllocateSpace(
if (bytes_limit_ != -1
&& current_bytes_allocated_ + scratch_range_bytes > bytes_limit_) {
- return Status(TErrorCode::SCRATCH_LIMIT_EXCEEDED, bytes_limit_);
+ return Status(TErrorCode::SCRATCH_LIMIT_EXCEEDED, bytes_limit_, GetBackendString());
}
// Lazily create the files on the first write.
@@ -331,7 +332,8 @@ Status TmpFileMgr::FileGroup::AllocateSpace(
<< ". Will try another scratch file.";
scratch_errors_.push_back(status);
}
- Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED);
+ Status err_status(TErrorCode::SCRATCH_ALLOCATION_FAILED,
+ join(tmp_file_mgr_->tmp_dirs_, ","), GetBackendString());
// Include all previous errors that may have caused the failure.
for (Status& err : scratch_errors_) err_status.MergeStatus(err);
return err_status;
@@ -410,7 +412,7 @@ Status TmpFileMgr::FileGroup::WaitForAsyncRead(WriteHandle* handle, MemRange buf
if (io_mgr_buffer->len() < buffer.len()) {
// The read was truncated - this is an error.
status = Status(TErrorCode::SCRATCH_READ_TRUNCATED, buffer.len(),
- handle->write_range_->file(), handle->write_range_->offset(),
+ handle->write_range_->file(), GetBackendString(), handle->write_range_->offset(),
io_mgr_buffer->len());
goto exit;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/39f23bb8/common/thrift/generate_error_codes.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index c08268e..3c37bef 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -275,7 +275,7 @@ error_codes = (
"supported length of 2147483647 bytes."),
("SCRATCH_LIMIT_EXCEEDED", 90, "Scratch space limit of $0 bytes exceeded for query "
- "while spilling data to disk."),
+ "while spilling data to disk on backend $1."),
("BUFFER_ALLOCATION_FAILED", 91, "Unexpected error allocating $0 byte buffer: $1"),
@@ -306,11 +306,11 @@ error_codes = (
# TODO: IMPALA-4697: the merged errors do not show up in the query error log,
# so we must point users to the impalad error log.
("SCRATCH_ALLOCATION_FAILED", 101, "Could not create files in any configured scratch "
- "directories (--scratch_dirs). See logs for previous errors that may have prevented "
- "creating or writing scratch files."),
+ "directories (--scratch_dirs=$0) on backend '$1'. See logs for previous errors that may "
+ "have prevented creating or writing scratch files."),
- ("SCRATCH_READ_TRUNCATED", 102, "Error reading $0 bytes from scratch file '$1' at "
- "offset $2: could only read $3 bytes"),
+ ("SCRATCH_READ_TRUNCATED", 102, "Error reading $0 bytes from scratch file '$1' "
+ "on backend $2 at offset $3: could only read $4 bytes"),
("KUDU_TIMESTAMP_OUT_OF_RANGE", 103,
"Kudu table '$0' column '$1' contains an out of range timestamp. "