You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/02/23 17:22:26 UTC
[1/3] impala git commit: KUDU-2305: Fix local variable usage to
handle 2GB messages
Repository: impala
Updated Branches:
refs/heads/master ad7cbc94e -> 1765a44da
KUDU-2305: Fix local variable usage to handle 2GB messages
The maximum value for rpc_max_message_size (an int32_t) is
INT_MAX. However, when using this maximum value, certain
local variables in SerializeMessage() and InboundTransfer
can overflow if the message size approaches INT_MAX.
This changes certain variables to handle messages that
approach INT_MAX. Specifically:
- Local variables in SerializeMessage() become int64_t
rather than int.
- The total message size prefixed to the message is now
treated as a uint32_t everywhere. This impacts
InboundTransfer::total_length_/cur_offset_ and
a local variable in ParseMessage().
These changes mean that a sender can serialize a message
that is slightly larger than INT_MAX due to the added
header size, but the receiver will reject all messages
exceeding rpc_max_message_size (maximum INT_MAX).
For testing, this modifies rpc-test's TestRpcSidecarLimits
to send a message of size INT_MAX rather than
rpc_max_message_size+1. This increases the test runtime
from about 50ms to 2 seconds.
Change-Id: I8e468099b16f7eb55de71b753acae8f57d18287c
Reviewed-on: http://gerrit.cloudera.org:8080/9355
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Reviewed-by: Dan Burkert <da...@cloudera.com>
Reviewed-on: http://gerrit.cloudera.org:8080/9407
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/152cf8b5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/152cf8b5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/152cf8b5
Branch: refs/heads/master
Commit: 152cf8b54d155cd4699b96f6fa83190f6ac9e9aa
Parents: ad7cbc9
Author: Joe McDonnell <jo...@cloudera.com>
Authored: Thu Feb 22 14:53:35 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Feb 23 06:58:40 2018 +0000
----------------------------------------------------------------------
be/src/kudu/rpc/rpc-test.cc | 6 +++++-
be/src/kudu/rpc/serialization.cc | 13 ++++++++-----
be/src/kudu/rpc/transfer.cc | 5 ++++-
be/src/kudu/rpc/transfer.h | 4 ++--
4 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/152cf8b5/be/src/kudu/rpc/rpc-test.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/rpc/rpc-test.cc b/be/src/kudu/rpc/rpc-test.cc
index 6d9d156..f76cc8f 100644
--- a/be/src/kudu/rpc/rpc-test.cc
+++ b/be/src/kudu/rpc/rpc-test.cc
@@ -17,6 +17,7 @@
#include "kudu/rpc/rpc-test-base.h"
+#include <limits.h>
#include <memory>
#include <string>
#include <unordered_map>
@@ -613,7 +614,10 @@ TEST_P(TestRpc, TestRpcSidecarLimits) {
GenericCalculatorService::static_service_name());
RpcController controller;
- string s(FLAGS_rpc_max_message_size + 1, 'a');
+ // KUDU-2305: Test with a maximal payload to verify that the implementation
+ // can handle the limits.
+ string s;
+ s.resize(INT_MAX, 'a');
int idx;
CHECK_OK(controller.AddOutboundSidecar(RpcSidecar::FromSlice(Slice(s)), &idx));
http://git-wip-us.apache.org/repos/asf/impala/blob/152cf8b5/be/src/kudu/rpc/serialization.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/rpc/serialization.cc b/be/src/kudu/rpc/serialization.cc
index fbc05bc..c31e79b 100644
--- a/be/src/kudu/rpc/serialization.cc
+++ b/be/src/kudu/rpc/serialization.cc
@@ -51,9 +51,12 @@ void SerializeMessage(const MessageLite& message, faststring* param_buf,
int additional_size, bool use_cached_size) {
int pb_size = use_cached_size ? message.GetCachedSize() : message.ByteSize();
DCHECK_EQ(message.ByteSize(), pb_size);
- int recorded_size = pb_size + additional_size;
- int size_with_delim = pb_size + CodedOutputStream::VarintSize32(recorded_size);
- int total_size = size_with_delim + additional_size;
+ // Use 8-byte integers to avoid overflowing when additional_size approaches INT_MAX.
+ int64_t recorded_size = static_cast<int64_t>(pb_size) +
+ static_cast<int64_t>(additional_size);
+ int64_t size_with_delim = static_cast<int64_t>(pb_size) +
+ static_cast<int64_t>(CodedOutputStream::VarintSize32(recorded_size));
+ int64_t total_size = size_with_delim + static_cast<int64_t>(additional_size);
if (total_size > FLAGS_rpc_max_message_size) {
LOG(WARNING) << Substitute("Serialized $0 ($1 bytes) is larger than the maximum configured "
@@ -109,8 +112,8 @@ Status ParseMessage(const Slice& buf,
KUDU_REDACT(buf.ToDebugString()));
}
- int total_len = NetworkByteOrder::Load32(buf.data());
- DCHECK_EQ(total_len + kMsgLengthPrefixLength, buf.size())
+ uint32_t total_len = NetworkByteOrder::Load32(buf.data());
+ DCHECK_EQ(total_len, buf.size() - kMsgLengthPrefixLength)
<< "Got mis-sized buffer: " << KUDU_REDACT(buf.ToDebugString());
CodedInputStream in(buf.data(), buf.size());
http://git-wip-us.apache.org/repos/asf/impala/blob/152cf8b5/be/src/kudu/rpc/transfer.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/rpc/transfer.cc b/be/src/kudu/rpc/transfer.cc
index da44355..f83c432 100644
--- a/be/src/kudu/rpc/transfer.cc
+++ b/be/src/kudu/rpc/transfer.cc
@@ -76,7 +76,7 @@ InboundTransfer::InboundTransfer()
Status InboundTransfer::ReceiveBuffer(Socket &socket) {
if (cur_offset_ < kMsgLengthPrefixLength) {
- // receive int32 length prefix
+ // receive uint32 length prefix
int32_t rem = kMsgLengthPrefixLength - cur_offset_;
int32_t nread;
Status status = socket.Recv(&buf_[cur_offset_], rem, &nread);
@@ -115,6 +115,9 @@ Status InboundTransfer::ReceiveBuffer(Socket &socket) {
// receive message body
int32_t nread;
+
+ // If total_length_ > INT_MAX, then it would exceed the maximum rpc_max_message_size
+ // and exit above. Hence, it is safe to use int32_t here.
int32_t rem = total_length_ - cur_offset_;
Status status = socket.Recv(&buf_[cur_offset_], rem, &nread);
RETURN_ON_ERROR_OR_SOCKET_NOT_READY(status);
http://git-wip-us.apache.org/repos/asf/impala/blob/152cf8b5/be/src/kudu/rpc/transfer.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/rpc/transfer.h b/be/src/kudu/rpc/transfer.h
index 2a2b726..533fe83 100644
--- a/be/src/kudu/rpc/transfer.h
+++ b/be/src/kudu/rpc/transfer.h
@@ -93,8 +93,8 @@ class InboundTransfer {
faststring buf_;
- int32_t total_length_;
- int32_t cur_offset_;
+ uint32_t total_length_;
+ uint32_t cur_offset_;
DISALLOW_COPY_AND_ASSIGN(InboundTransfer);
};
[2/3] impala git commit: IMPALA-6537: Add missing ODBC scalar
functions
Posted by ta...@apache.org.
IMPALA-6537: Add missing ODBC scalar functions
This patch contains the following builtin function changes:
New aliases for existing functions:
- LEFT() same as STRLEFT()
- RIGHT() same as STRRIGHT()
- WEEK() same as WEEKOFYEAR()
New functions:
- QUARTER()
- MONTHNAME()
Refactors:
- Remove TimestampFunctions::DayName and add LongDayName to match pattern of
TimestampFunctions::ShortDayName
Additionally, it adds the unit of QUARTER to EXTRACT() and DATE_PART()
Testing:
- manual testing comparing the translated ODBC functions to the
non-translated ones
- added at least one new expr-test for aliases
- new expr-tests added for new functions
Change-Id: Ia60af2b4de8c098be7ecb3e60840e459ae10d499
Reviewed-on: http://gerrit.cloudera.org:8080/9376
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d91df9b6
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d91df9b6
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d91df9b6
Branch: refs/heads/master
Commit: d91df9b63fa1fc8a907fc28e5db53b8eb275e8f3
Parents: 152cf8b
Author: Greg Rahn <gr...@users.noreply.github.com>
Authored: Tue Feb 20 18:31:57 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Feb 23 07:19:07 2018 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 42 +++++++++++++++++++
be/src/exprs/timestamp-functions-ir.cc | 43 ++++++++++++--------
be/src/exprs/timestamp-functions.cc | 14 +++----
be/src/exprs/timestamp-functions.h | 21 ++++------
be/src/exprs/udf-builtins-ir.cc | 6 +++
common/function-registry/impala_functions.py | 11 ++---
common/thrift/Exprs.thrift | 1 +
fe/src/main/cup/sql-parser.cup | 6 ++-
.../impala/analysis/AnalyzeExprsTest.java | 2 +-
.../org/apache/impala/analysis/ParserTest.java | 17 ++++----
10 files changed, 109 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index a78fcae..71f91c8 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -3759,6 +3759,7 @@ TEST_F(ExprTest, StringFunctions) {
TestIsNull("strleft(NULL, 3)", TYPE_STRING);
TestIsNull("strleft('abcdefg', NULL)", TYPE_STRING);
TestIsNull("strleft(NULL, NULL)", TYPE_STRING);
+ TestStringValue("left('foobar', 3)", "foo");
TestStringValue("strright('abcdefg', 0)", "");
TestStringValue("strright('abcdefg', 3)", "efg");
TestStringValue("strright('abcdefg', cast(10 as bigint))", "abcdefg");
@@ -3767,6 +3768,7 @@ TEST_F(ExprTest, StringFunctions) {
TestIsNull("strright(NULL, 3)", TYPE_STRING);
TestIsNull("strright('abcdefg', NULL)", TYPE_STRING);
TestIsNull("strright(NULL, NULL)", TYPE_STRING);
+ TestStringValue("right('foobar', 3)", "bar");
TestStringValue("translate('', '', '')", "");
TestStringValue("translate('abcd', '', '')", "abcd");
@@ -5933,11 +5935,13 @@ TEST_F(ExprTest, TimestampFunctions) {
TestValue("cast('2011-12-22 09:10:11.000000' as timestamp) = \
cast('2011-12-22 09:10:11' as timestamp)", TYPE_BOOLEAN, true);
TestValue("year(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 2011);
+ TestValue("quarter(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 4);
TestValue("month(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 12);
TestValue("dayofmonth(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 22);
TestValue("day(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 22);
TestValue("dayofyear(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 356);
TestValue("weekofyear(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 51);
+ TestValue("week(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 51);
TestValue("dayofweek(cast('2011-12-18 09:10:11.000000' as timestamp))", TYPE_INT, 1);
TestValue("dayofweek(cast('2011-12-22 09:10:11.000000' as timestamp))", TYPE_INT, 5);
TestValue("dayofweek(cast('2011-12-24 09:10:11.000000' as timestamp))", TYPE_INT, 7);
@@ -5949,11 +5953,13 @@ TEST_F(ExprTest, TimestampFunctions) {
TestValue("millisecond(cast('2011-12-22 09:10:11' as timestamp))", TYPE_INT, 0);
TestValue("millisecond(cast('2011-12-22' as timestamp))", TYPE_INT, 0);
TestValue("year(cast('2011-12-22' as timestamp))", TYPE_INT, 2011);
+ TestValue("quarter(cast('2011-12-22' as timestamp))", TYPE_INT, 4);
TestValue("month(cast('2011-12-22' as timestamp))", TYPE_INT, 12);
TestValue("dayofmonth(cast('2011-12-22' as timestamp))", TYPE_INT, 22);
TestValue("day(cast('2011-12-22' as timestamp))", TYPE_INT, 22);
TestValue("dayofyear(cast('2011-12-22' as timestamp))", TYPE_INT, 356);
TestValue("weekofyear(cast('2011-12-22' as timestamp))", TYPE_INT, 51);
+ TestValue("week(cast('2011-12-22' as timestamp))", TYPE_INT, 51);
TestValue("dayofweek(cast('2011-12-18' as timestamp))", TYPE_INT, 1);
TestValue("dayofweek(cast('2011-12-22' as timestamp))", TYPE_INT, 5);
TestValue("dayofweek(cast('2011-12-24' as timestamp))", TYPE_INT, 7);
@@ -6011,22 +6017,26 @@ TEST_F(ExprTest, TimestampFunctions) {
TestIsNull("cast('2000-12-31 24:59:59' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("year(cast('09:10:11.000000' as timestamp))", TYPE_INT);
+ TestIsNull("quarter(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("month(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("dayofmonth(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("day(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("dayofyear(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("dayofweek(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("weekofyear(cast('09:10:11.000000' as timestamp))", TYPE_INT);
+ TestIsNull("week(cast('09:10:11.000000' as timestamp))", TYPE_INT);
TestIsNull("datediff(cast('09:10:11.12345678' as timestamp), "
"cast('2012-12-22' as timestamp))", TYPE_INT);
TestIsNull("year(NULL)", TYPE_INT);
+ TestIsNull("quarter(NULL)", TYPE_INT);
TestIsNull("month(NULL)", TYPE_INT);
TestIsNull("dayofmonth(NULL)", TYPE_INT);
TestIsNull("day(NULL)", TYPE_INT);
TestIsNull("dayofweek(NULL)", TYPE_INT);
TestIsNull("dayofyear(NULL)", TYPE_INT);
TestIsNull("weekofyear(NULL)", TYPE_INT);
+ TestIsNull("week(NULL)", TYPE_INT);
TestIsNull("datediff(NULL, cast('2011-12-22 09:10:11.12345678' as timestamp))",
TYPE_INT);
TestIsNull("datediff(cast('2012-12-22' as timestamp), NULL)", TYPE_INT);
@@ -6046,6 +6056,34 @@ TEST_F(ExprTest, TimestampFunctions) {
TestStringValue("dayname(cast('2011-12-25 09:10:11.000000' as timestamp))", "Sunday");
TestIsNull("dayname(NULL)", TYPE_STRING);
+ TestStringValue("monthname(cast('2011-01-18 09:10:11.000000' as timestamp))", "January");
+ TestStringValue("monthname(cast('2011-02-18 09:10:11.000000' as timestamp))", "February");
+ TestStringValue("monthname(cast('2011-03-18 09:10:11.000000' as timestamp))", "March");
+ TestStringValue("monthname(cast('2011-04-18 09:10:11.000000' as timestamp))", "April");
+ TestStringValue("monthname(cast('2011-05-18 09:10:11.000000' as timestamp))", "May");
+ TestStringValue("monthname(cast('2011-06-18 09:10:11.000000' as timestamp))", "June");
+ TestStringValue("monthname(cast('2011-07-18 09:10:11.000000' as timestamp))", "July");
+ TestStringValue("monthname(cast('2011-08-18 09:10:11.000000' as timestamp))", "August");
+ TestStringValue("monthname(cast('2011-09-18 09:10:11.000000' as timestamp))", "September");
+ TestStringValue("monthname(cast('2011-10-18 09:10:11.000000' as timestamp))", "October");
+ TestStringValue("monthname(cast('2011-11-18 09:10:11.000000' as timestamp))", "November");
+ TestStringValue("monthname(cast('2011-12-18 09:10:11.000000' as timestamp))", "December");
+ TestIsNull("monthname(NULL)", TYPE_STRING);
+
+ TestValue("quarter(cast('2011-01-18 09:10:11.000000' as timestamp))", TYPE_INT, 1);
+ TestValue("quarter(cast('2011-02-18 09:10:11.000000' as timestamp))", TYPE_INT, 1);
+ TestValue("quarter(cast('2011-03-18 09:10:11.000000' as timestamp))", TYPE_INT, 1);
+ TestValue("quarter(cast('2011-04-18 09:10:11.000000' as timestamp))", TYPE_INT, 2);
+ TestValue("quarter(cast('2011-05-18 09:10:11.000000' as timestamp))", TYPE_INT, 2);
+ TestValue("quarter(cast('2011-06-18 09:10:11.000000' as timestamp))", TYPE_INT, 2);
+ TestValue("quarter(cast('2011-07-18 09:10:11.000000' as timestamp))", TYPE_INT, 3);
+ TestValue("quarter(cast('2011-08-18 09:10:11.000000' as timestamp))", TYPE_INT, 3);
+ TestValue("quarter(cast('2011-09-18 09:10:11.000000' as timestamp))", TYPE_INT, 3);
+ TestValue("quarter(cast('2011-10-18 09:10:11.000000' as timestamp))", TYPE_INT, 4);
+ TestValue("quarter(cast('2011-11-18 09:10:11.000000' as timestamp))", TYPE_INT, 4);
+ TestValue("quarter(cast('2011-12-18 09:10:11.000000' as timestamp))", TYPE_INT, 4);
+ TestIsNull("quarter(NULL)", TYPE_INT);
+
// Tests from Hive
// The hive documentation states that timestamps are timezoneless, but the tests
// show that they treat them as being in the current timezone so these tests
@@ -6526,6 +6564,8 @@ TEST_F(ExprTest, TimestampFunctions) {
// Extract using FROM keyword
TestValue("extract(YEAR from cast('2006-05-12 18:27:28.12345' as timestamp))",
TYPE_INT, 2006);
+ TestValue("extract(QUARTER from cast('2006-05-12 18:27:28.12345' as timestamp))",
+ TYPE_INT, 2);
TestValue("extract(MoNTH from cast('2006-05-12 18:27:28.12345' as timestamp))",
TYPE_INT, 5);
TestValue("extract(DaY from cast('2006-05-12 18:27:28.12345' as timestamp))",
@@ -6547,6 +6587,8 @@ TEST_F(ExprTest, TimestampFunctions) {
// Date_part, same as extract function but with arguments swapped
TestValue("date_part('YEAR', cast('2006-05-12 18:27:28.12345' as timestamp))",
TYPE_INT, 2006);
+ TestValue("date_part('QUARTER', cast('2006-05-12 18:27:28.12345' as timestamp))",
+ TYPE_INT, 2);
TestValue("date_part('MoNTH', cast('2006-05-12 18:27:28.12345' as timestamp))",
TYPE_INT, 5);
TestValue("date_part('DaY', cast('2006-05-12 18:27:28.12345' as timestamp))",
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 9104bcf..f2d741e 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -204,21 +204,6 @@ void TimestampFunctions::ReportBadFormat(FunctionContext* context,
}
}
-StringVal TimestampFunctions::DayName(FunctionContext* context, const TimestampVal& ts) {
- if (ts.is_null) return StringVal::null();
- IntVal dow = DayOfWeek(context, ts);
- switch(dow.val) {
- case 1: return StringVal(SUNDAY);
- case 2: return StringVal(MONDAY);
- case 3: return StringVal(TUESDAY);
- case 4: return StringVal(WEDNESDAY);
- case 5: return StringVal(THURSDAY);
- case 6: return StringVal(FRIDAY);
- case 7: return StringVal(SATURDAY);
- default: return StringVal::null();
- }
-}
-
IntVal TimestampFunctions::Year(FunctionContext* context, const TimestampVal& ts_val) {
if (ts_val.is_null) return IntVal::null();
const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
@@ -226,7 +211,6 @@ IntVal TimestampFunctions::Year(FunctionContext* context, const TimestampVal& ts
return IntVal(ts_value.date().year());
}
-
IntVal TimestampFunctions::Month(FunctionContext* context, const TimestampVal& ts_val) {
if (ts_val.is_null) return IntVal::null();
const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
@@ -234,6 +218,13 @@ IntVal TimestampFunctions::Month(FunctionContext* context, const TimestampVal& t
return IntVal(ts_value.date().month());
}
+IntVal TimestampFunctions::Quarter(FunctionContext* context, const TimestampVal& ts_val) {
+ if (ts_val.is_null) return IntVal::null();
+ const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (!ts_value.HasDate()) return IntVal::null();
+ int m = ts_value.date().month();
+ return IntVal((m - 1) / 3 + 1);
+}
IntVal TimestampFunctions::DayOfWeek(FunctionContext* context,
const TimestampVal& ts_val) {
@@ -473,6 +464,16 @@ string TimestampFunctions::ShortDayName(FunctionContext* context,
return DAY_ARRAY[dow.val - 1];
}
+StringVal TimestampFunctions::LongDayName(FunctionContext* context,
+ const TimestampVal& ts) {
+ if (ts.is_null) return StringVal::null();
+ IntVal dow = DayOfWeek(context, ts);
+ DCHECK_GT(dow.val, 0);
+ DCHECK_LT(dow.val, 8);
+ const string& day_name = DAYNAME_ARRAY[dow.val - 1];
+ return StringVal(reinterpret_cast<uint8_t*>(const_cast<char*>(day_name.data())), day_name.size());
+}
+
string TimestampFunctions::ShortMonthName(FunctionContext* context,
const TimestampVal& ts) {
if (ts.is_null) return NULL;
@@ -482,6 +483,16 @@ string TimestampFunctions::ShortMonthName(FunctionContext* context,
return MONTH_ARRAY[mth.val - 1];
}
+StringVal TimestampFunctions::LongMonthName(FunctionContext* context,
+ const TimestampVal& ts) {
+ if (ts.is_null) return StringVal::null();
+ IntVal mth = Month(context, ts);
+ DCHECK_GT(mth.val, 0);
+ DCHECK_LT(mth.val, 13);
+ const string& mn = MONTHNAME_ARRAY[mth.val - 1];
+ return StringVal(reinterpret_cast<uint8_t*>(const_cast<char*>(mn.data())), mn.size());
+}
+
StringVal TimestampFunctions::TimeOfDay(FunctionContext* context) {
const TimestampVal curr = Now(context);
if (curr.is_null) return StringVal::null();
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/be/src/exprs/timestamp-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
index d9372fb..76edd9a 100644
--- a/be/src/exprs/timestamp-functions.cc
+++ b/be/src/exprs/timestamp-functions.cc
@@ -42,19 +42,15 @@ typedef boost::gregorian::date Date;
namespace impala {
-// Constant strings used for DayName function.
-const char* TimestampFunctions::SUNDAY = "Sunday";
-const char* TimestampFunctions::MONDAY = "Monday";
-const char* TimestampFunctions::TUESDAY = "Tuesday";
-const char* TimestampFunctions::WEDNESDAY = "Wednesday";
-const char* TimestampFunctions::THURSDAY = "Thursday";
-const char* TimestampFunctions::FRIDAY = "Friday";
-const char* TimestampFunctions::SATURDAY = "Saturday";
-
const string TimestampFunctions::DAY_ARRAY[7] = {"Sun", "Mon", "Tue", "Wed", "Thu",
"Fri", "Sat"};
+const string TimestampFunctions::DAYNAME_ARRAY[7] = {"Sunday", "Monday", "Tuesday",
+ "Wednesday", "Thursday", "Friday", "Saturday"};
const string TimestampFunctions::MONTH_ARRAY[12] = {"Jan", "Feb", "Mar", "Apr", "May",
"Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
+const string TimestampFunctions::MONTHNAME_ARRAY[12] = {"January", "February", "March",
+ "April", "May", "June", "July", "August", "September", "October", "November",
+ "December"};
namespace {
/// Uses Boost's internal checking to throw an exception if 'date' is out of the
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/be/src/exprs/timestamp-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index fdf33bd..ee0d54f 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -123,11 +123,9 @@ class TimestampFunctions {
static TimestampVal ToUtc(FunctionContext* context,
const TimestampVal& ts_val, const StringVal& tz_string_val);
- /// Returns the day's name as a string (e.g. 'Saturday').
- static StringVal DayName(FunctionContext* context, const TimestampVal& dow);
-
/// Functions to extract parts of the timestamp, return integers.
static IntVal Year(FunctionContext* context, const TimestampVal& ts_val);
+ static IntVal Quarter(FunctionContext* context, const TimestampVal& ts_val);
static IntVal Month(FunctionContext* context, const TimestampVal& ts_val);
static IntVal DayOfWeek(FunctionContext* context, const TimestampVal& ts_val);
static IntVal DayOfMonth(FunctionContext* context, const TimestampVal& ts_val);
@@ -145,7 +143,9 @@ class TimestampFunctions {
static IntVal DateDiff(FunctionContext* context, const TimestampVal& ts_val1,
const TimestampVal& ts_val2);
static std::string ShortDayName(FunctionContext* context, const TimestampVal& ts);
+ static StringVal LongDayName(FunctionContext* context, const TimestampVal& ts);
static std::string ShortMonthName(FunctionContext* context, const TimestampVal& ts);
+ static StringVal LongMonthName(FunctionContext* context, const TimestampVal& ts);
/// Return verbose string version of current time of day
/// e.g. Mon Dec 01 16:25:05 2003 EST.
@@ -229,18 +229,13 @@ class TimestampFunctions {
const StringVal& format, bool is_error);
private:
- /// Static result values for DayName() function.
- static const char* MONDAY;
- static const char* TUESDAY;
- static const char* WEDNESDAY;
- static const char* THURSDAY;
- static const char* FRIDAY;
- static const char* SATURDAY;
- static const char* SUNDAY;
-
- /// Static result values for ShortDayName() and ShortMonthName() functions.
+
+ /// Static result values for DayName(), ShortDayName(), ShortMonthName() and
+ /// LongMonthName functions.
static const std::string DAY_ARRAY[7];
+ static const std::string DAYNAME_ARRAY[7];
static const std::string MONTH_ARRAY[12];
+ static const std::string MONTHNAME_ARRAY[12];
};
} // namespace impala
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/be/src/exprs/udf-builtins-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/udf-builtins-ir.cc b/be/src/exprs/udf-builtins-ir.cc
index 03b88ba..3c47e6b 100644
--- a/be/src/exprs/udf-builtins-ir.cc
+++ b/be/src/exprs/udf-builtins-ir.cc
@@ -119,6 +119,7 @@ TExtractField::type StrToExtractField(FunctionContext* ctx, const StringVal& uni
StringVal unit = UdfBuiltins::Lower(ctx, unit_str);
if (UNLIKELY(unit.is_null)) return TExtractField::INVALID_FIELD;
if (unit == "year") return TExtractField::YEAR;
+ if (unit == "quarter") return TExtractField::QUARTER;
if (unit == "month") return TExtractField::MONTH;
if (unit == "day") return TExtractField::DAY;
if (unit == "hour") return TExtractField::HOUR;
@@ -153,6 +154,7 @@ IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
switch (field) {
case TExtractField::YEAR:
+ case TExtractField::QUARTER:
case TExtractField::MONTH:
case TExtractField::DAY:
if (orig_date.is_special()) return IntVal::null();
@@ -174,6 +176,10 @@ IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
case TExtractField::YEAR: {
return IntVal(orig_date.year());
}
+ case TExtractField::QUARTER: {
+ int m = orig_date.month();
+ return IntVal((m - 1) / 3 + 1);
+ }
case TExtractField::MONTH: {
return IntVal(orig_date.month());
}
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/common/function-registry/impala_functions.py
----------------------------------------------------------------------
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index 8174abb..b3387c6 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -109,21 +109,23 @@ visible_functions = [
'_ZN6impala11UdfBuiltins9VectorGetEPN10impala_udf15FunctionContextERKNS1_9BigIntValERKNS1_9StringValE'],
# Timestamp functions
+ [['monthname'], 'STRING', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions13LongMonthNameEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['next_day'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'], '_ZN6impala18TimestampFunctions7NextDayEPN10impala_udf15FunctionContextERKNS1_12TimestampValERKNS1_9StringValE'],
[['last_day'], 'TIMESTAMP', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions7LastDayEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['unix_timestamp'], 'BIGINT', ['STRING'], '_ZN6impala18TimestampFunctions14UnixFromStringEPN10impala_udf15FunctionContextERKNS1_9StringValE'],
[['year'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions4YearEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
+ [['quarter'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions7QuarterEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['month'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions5MonthEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['dayofweek'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions9DayOfWeekEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['day', 'dayofmonth'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions10DayOfMonthEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['dayofyear'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions9DayOfYearEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
- [['weekofyear'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions10WeekOfYearEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
+ [['week', 'weekofyear'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions10WeekOfYearEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['hour'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions4HourEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['minute'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions6MinuteEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['second'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions6SecondEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['millisecond'], 'INT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions11MillisecondEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['to_date'], 'STRING', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions6ToDateEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
- [['dayname'], 'STRING', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions7DayNameEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
+ [['dayname'], 'STRING', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions11LongDayNameEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['date_trunc'], 'TIMESTAMP', ['STRING', 'TIMESTAMP'],
'_ZN6impala11UdfBuiltins9DateTruncEPN10impala_udf15FunctionContextERKNS1_9StringValERKNS1_12TimestampValE',
'_ZN6impala11UdfBuiltins16DateTruncPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
@@ -419,9 +421,8 @@ visible_functions = [
'impala::StringFunctions::SplitPart'],
[['base64encode'], 'STRING', ['STRING'], 'impala::StringFunctions::Base64Encode'],
[['base64decode'], 'STRING', ['STRING'], 'impala::StringFunctions::Base64Decode'],
- # left and right are key words, leave them out for now.
- [['strleft'], 'STRING', ['STRING', 'BIGINT'], 'impala::StringFunctions::Left'],
- [['strright'], 'STRING', ['STRING', 'BIGINT'], 'impala::StringFunctions::Right'],
+ [['left', 'strleft'], 'STRING', ['STRING', 'BIGINT'], 'impala::StringFunctions::Left'],
+ [['right', 'strright'], 'STRING', ['STRING', 'BIGINT'], 'impala::StringFunctions::Right'],
[['space'], 'STRING', ['BIGINT'], 'impala::StringFunctions::Space'],
[['repeat'], 'STRING', ['STRING', 'BIGINT'], 'impala::StringFunctions::Repeat'],
[['lpad'], 'STRING', ['STRING', 'BIGINT', 'STRING'], 'impala::StringFunctions::Lpad'],
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/common/thrift/Exprs.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Exprs.thrift b/common/thrift/Exprs.thrift
index 7c88f2b..ee9fe94 100644
--- a/common/thrift/Exprs.thrift
+++ b/common/thrift/Exprs.thrift
@@ -79,6 +79,7 @@ struct TTimestampLiteral {
enum TExtractField {
INVALID_FIELD,
YEAR,
+ QUARTER,
MONTH,
DAY,
HOUR,
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 9f55dda..9802778 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -2799,13 +2799,17 @@ non_pred_expr ::=
{: RESULT = e; :}
| analytic_expr:e
{: RESULT = e; :}
- /* Since "IF", "REPLACE", "TRUNCATE" are keywords, need to special case these functions */
+ /* Additional rules for function names that are also keywords */
| KW_IF LPAREN expr_list:exprs RPAREN
{: RESULT = new FunctionCallExpr("if", exprs); :}
| KW_REPLACE LPAREN expr_list:exprs RPAREN
{: RESULT = new FunctionCallExpr("replace", exprs); :}
| KW_TRUNCATE LPAREN expr_list:exprs RPAREN
{: RESULT = new FunctionCallExpr("truncate", exprs); :}
+ | KW_LEFT LPAREN expr_list:exprs RPAREN
+ {: RESULT = new FunctionCallExpr("left", exprs); :}
+ | KW_RIGHT LPAREN expr_list:exprs RPAREN
+ {: RESULT = new FunctionCallExpr("right", exprs); :}
| cast_expr:c
{: RESULT = c; :}
| case_expr:c
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index d3eeb19..8f2cc60 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -1786,7 +1786,7 @@ public class AnalyzeExprsTest extends AnalyzerTest {
AnalyzesOk("select extract(year from now())");
AnalysisError("select extract(foo from now())",
"Time unit 'foo' in expression 'EXTRACT(foo FROM now())' is invalid. Expected " +
- "one of YEAR, MONTH, DAY, HOUR, MINUTE, SECOND, MILLISECOND, EPOCH.");
+ "one of YEAR, QUARTER, MONTH, DAY, HOUR, MINUTE, SECOND, MILLISECOND, EPOCH.");
AnalysisError("select extract(year from 0)",
"Expression '0' in 'EXTRACT(year FROM 0)' has a return type of TINYINT but a " +
"TIMESTAMP is required.");
http://git-wip-us.apache.org/repos/asf/impala/blob/d91df9b6/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 178539f..b51d148 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -3243,9 +3243,8 @@ public class ParserTest extends FrontendTestBase {
"select from t\n" +
" ^\n" +
"Encountered: FROM\n" +
- "Expected: ALL, CASE, CAST, DEFAULT, DISTINCT, EXISTS, " +
- "FALSE, IF, INTERVAL, NOT, NULL, REPLACE, " +
- "STRAIGHT_JOIN, TRUNCATE, TRUE, IDENTIFIER\n");
+ "Expected: ALL, CASE, CAST, DEFAULT, DISTINCT, EXISTS, FALSE, IF, INTERVAL, " +
+ "LEFT, NOT, NULL, REPLACE, RIGHT, STRAIGHT_JOIN, TRUNCATE, TRUE, IDENTIFIER");
// missing from
ParserError("select c, b, c where a = 5",
@@ -3270,8 +3269,8 @@ public class ParserTest extends FrontendTestBase {
"select c, b, c from t where\n" +
" ^\n" +
"Encountered: EOF\n" +
- "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, " +
- "IF, INTERVAL, NOT, NULL, REPLACE, TRUNCATE, TRUE, IDENTIFIER\n");
+ "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, IF, INTERVAL, LEFT, NOT, NULL, " +
+ "REPLACE, RIGHT, TRUNCATE, TRUE, IDENTIFIER");
// missing predicate in where clause (group by)
ParserError("select c, b, c from t where group by a, b",
@@ -3279,8 +3278,8 @@ public class ParserTest extends FrontendTestBase {
"select c, b, c from t where group by a, b\n" +
" ^\n" +
"Encountered: GROUP\n" +
- "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, " +
- "IF, INTERVAL, NOT, NULL, REPLACE, TRUNCATE, TRUE, IDENTIFIER\n");
+ "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, IF, INTERVAL, LEFT, NOT, NULL, " +
+ "REPLACE, RIGHT, TRUNCATE, TRUE, IDENTIFIER");
// unmatched string literal starting with "
ParserError("select c, \"b, c from t",
@@ -3340,8 +3339,8 @@ public class ParserTest extends FrontendTestBase {
"...c,c,c,c,c,c,c,c,cd,c,d,d, ,c, from t\n" +
" ^\n" +
"Encountered: COMMA\n" +
- "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, " +
- "IF, INTERVAL, NOT, NULL, REPLACE, TRUNCATE, TRUE, IDENTIFIER\n");
+ "Expected: CASE, CAST, DEFAULT, EXISTS, FALSE, IF, INTERVAL, LEFT, NOT, NULL, " +
+ "REPLACE, RIGHT, TRUNCATE, TRUE, IDENTIFIER");
// Parsing identifiers that have different names printed as EXPECTED
ParserError("DROP DATA SRC foo",
[3/3] impala git commit: IMPALA-6275: Fix warn stacktrace in
successful CTAS
Posted by ta...@apache.org.
IMPALA-6275: Fix warn stacktrace in successful CTAS
For CTAS in HDFS tables, do not load the column statistics from HMS
since since that information will not be available in HMS.
Refactored CTAS for HDFS tables to use HdfsTable.createCtasTarget()
instead of HdfsTable.load().
Testing:
- Ran end-to-end query tests
Change-Id: I6f07a188458954802fda20e3b3b56280d96e788e
Reviewed-on: http://gerrit.cloudera.org:8080/9364
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/1765a44d
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/1765a44d
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/1765a44d
Branch: refs/heads/master
Commit: 1765a44da00401298a6367a2885996f4b12a5e3b
Parents: d91df9b
Author: Fredy wijaya <fw...@cloudera.com>
Authored: Thu Feb 22 11:10:36 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Feb 23 10:49:28 2018 +0000
----------------------------------------------------------------------
.../analysis/CreateTableAsSelectStmt.java | 8 ++--
.../org/apache/impala/catalog/HdfsTable.java | 42 +++++++++++++-------
2 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/1765a44d/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
index 222b99a..aac6873 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.HdfsFileFormat;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
@@ -211,11 +212,8 @@ public class CreateTableAsSelectStmt extends StatementBase {
tmpTable = KuduTable.createCtasTarget(db, msTbl, createStmt_.getColumnDefs(),
createStmt_.getPrimaryKeyColumnDefs(),
createStmt_.getKuduPartitionParams());
- } else {
- // TODO: Creating a tmp table using load() is confusing.
- // Refactor it to use a 'createCtasTarget()' function similar to Kudu table.
- tmpTable = Table.fromMetastoreTable(db, msTbl);
- tmpTable.load(true, client.getHiveClient(), msTbl);
+ } else if (HdfsFileFormat.isHdfsInputFormatClass(msTbl.getSd().getInputFormat())) {
+ tmpTable = HdfsTable.createCtasTarget(db, msTbl);
}
Preconditions.checkState(tmpTable != null &&
(tmpTable instanceof HdfsTable || tmpTable instanceof KuduTable));
http://git-wip-us.apache.org/repos/asf/impala/blob/1765a44d/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 0f782be..14f2888 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -1246,13 +1247,12 @@ public class HdfsTable extends Table {
// turn all exceptions into TableLoadingException
msTable_ = msTbl;
try {
- if (loadTableSchema) loadSchema(client, msTbl);
- if (reuseMetadata && getCatalogVersion() == Catalog.INITIAL_CATALOG_VERSION) {
- // This is the special case of CTAS that creates a 'temp' table that does not
- // actually exist in the Hive Metastore.
- initializePartitionMetadata(msTbl);
- setTableStats(msTbl);
- return;
+ if (loadTableSchema) {
+ // set nullPartitionKeyValue from the hive conf.
+ nullPartitionKeyValue_ = client.getConfigValue(
+ "hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__");
+ loadSchema(msTbl);
+ loadAllColumnStats(client);
}
// Load partition and file metadata
if (reuseMetadata) {
@@ -1568,15 +1568,11 @@ public class HdfsTable extends Table {
}
/**
- * Loads table schema and column stats from Hive Metastore.
+ * Loads table schema.
*/
- private void loadSchema(IMetaStoreClient client,
- org.apache.hadoop.hive.metastore.api.Table msTbl) throws Exception {
+ private void loadSchema(org.apache.hadoop.hive.metastore.api.Table msTbl)
+ throws TableLoadingException {
nonPartFieldSchemas_.clear();
- // set nullPartitionKeyValue from the hive conf.
- nullPartitionKeyValue_ = client.getConfigValue(
- "hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__");
-
// set NULL indicator string from table properties
nullColumnValue_ =
msTbl.getParameters().get(serdeConstants.SERIALIZATION_NULL_FORMAT);
@@ -1593,7 +1589,6 @@ public class HdfsTable extends Table {
// then all other columns.
addColumnsFromFieldSchemas(msTbl.getPartitionKeys());
addColumnsFromFieldSchemas(nonPartFieldSchemas_);
- loadAllColumnStats(client);
isSchemaLoaded_ = true;
}
@@ -2278,4 +2273,21 @@ public class HdfsTable extends Table {
});
metrics_.addTimer(CATALOG_UPDATE_DURATION_METRIC);
}
+
+ /**
+ * Creates a temporary HdfsTable object populated with the specified properties.
+ * This is used for CTAS statements.
+ */
+ public static HdfsTable createCtasTarget(Db db,
+ org.apache.hadoop.hive.metastore.api.Table msTbl) throws CatalogException {
+ HdfsTable tmpTable = new HdfsTable(msTbl, db, msTbl.getTableName(), msTbl.getOwner());
+ HiveConf hiveConf = new HiveConf(HdfsTable.class);
+ // set nullPartitionKeyValue from the hive conf.
+ tmpTable.nullPartitionKeyValue_ = hiveConf.get(
+ "hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__");
+ tmpTable.loadSchema(msTbl);
+ tmpTable.initializePartitionMetadata(msTbl);
+ tmpTable.setTableStats(msTbl);
+ return tmpTable;
+ }
}