You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mj...@apache.org on 2016/08/10 16:05:30 UTC
[1/3] incubator-impala git commit: IMPALA-2700: ASCII NUL characters
are doubled on insert into text tables
Repository: incubator-impala
Updated Branches:
refs/heads/master 0d689d362 -> 27b3b4d47
IMPALA-2700: ASCII NUL characters are doubled on insert into text tables
Currently the scanner processes the '\0' character as a no special character
whereas the writer treats it as a special character. The writer
appends a special character before writting which is causing the ASCII
NULL characters to double since they are the default escape characters.
This adds a check to treat '\0' as a no special character in the
writter.
Change-Id: Ia30fa314d1ee1e99f9e7598466eb1570ca7940fc
Reviewed-on: http://gerrit.cloudera.org:8080/3876
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b66829f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b66829f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b66829f1
Branch: refs/heads/master
Commit: b66829f15f5d3072ba56b966ef2ca21620a59390
Parents: 0d689d3
Author: Anuj Phadke <ap...@cloudera.com>
Authored: Wed Jul 20 14:26:17 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Aug 10 04:09:38 2016 +0000
----------------------------------------------------------------------
be/src/exec/hdfs-text-table-writer.cc | 11 +++++++----
tests/query_test/test_insert_behaviour.py | 9 +++++++++
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b66829f1/be/src/exec/hdfs-text-table-writer.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-table-writer.cc b/be/src/exec/hdfs-text-table-writer.cc
index 59ecd9c..053c821 100644
--- a/be/src/exec/hdfs-text-table-writer.cc
+++ b/be/src/exec/hdfs-text-table-writer.cc
@@ -208,11 +208,14 @@ Status HdfsTextTableWriter::Flush() {
inline void HdfsTextTableWriter::PrintEscaped(const StringValue* str_val) {
for (int i = 0; i < str_val->len; ++i) {
- if (UNLIKELY(str_val->ptr[i] == field_delim_ || str_val->ptr[i] == escape_char_)) {
- rowbatch_stringstream_ << escape_char_;
+ if (escape_char_ == '\0') {
+ rowbatch_stringstream_ << str_val->ptr[i];
+ } else {
+ if (UNLIKELY(str_val->ptr[i] == field_delim_ || str_val->ptr[i] == escape_char_)) {
+ rowbatch_stringstream_ << escape_char_;
+ }
+ rowbatch_stringstream_ << str_val->ptr[i];
}
- rowbatch_stringstream_ << str_val->ptr[i];
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b66829f1/tests/query_test/test_insert_behaviour.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index 7dcb89d..53c9dbf 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -83,6 +83,15 @@ class TestInsertBehaviour(ImpalaTestSuite):
assert self.filesystem_client.exists(table_dir + dir_), "Directory {0} was " \
"unexpectedly deleted by INSERT OVERWRITE".format(table_dir + dir_)
+ def test_insert_ascii_nulls(self, unique_database):
+ TBL_NAME = '`{0}`.`null_insert`'.format(unique_database)
+ self.execute_query_expect_success(self.client, "DROP TABLE IF EXISTS %s" % TBL_NAME)
+ self.execute_query_expect_success(self.client, "create table %s as select '\0' s"
+ % TBL_NAME)
+ result = self.execute_query_expect_success(self.client,
+ "SELECT LENGTH(s) FROM %s" % TBL_NAME)
+ assert int(result.get_data()) == 1
+
@UniqueDatabase.parametrize(name_prefix='test_insert_alter_partition_location_db')
def test_insert_alter_partition_location(self, unique_database):
"""Test that inserts after changing the location of a partition work correctly,
[3/3] incubator-impala git commit: IMPALA-1112: Remove some
unncessary code from cross-compilation
Posted by mj...@apache.org.
IMPALA-1112: Remove some unncessary code from cross-compilation
This change stops including some boost library header files
which pulls in other unnecessary boost library header files.
This reduces the amount of cross-compiled code which needs
to be materialized during codegen.
This change also removes some UDF's Prepare() and Close()
functions and UDF functions fromUtc(), toUtc() and uuid()
from cross-compilation as they won't benefit from it.
With this change, the bitcode module reduces from 2.12 MB to 1.86MB.
Change-Id: I543809c69da0b4085a0e299b91cd550b274c46af
Reviewed-on: http://gerrit.cloudera.org:8080/3793
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/27b3b4d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/27b3b4d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/27b3b4d4
Branch: refs/heads/master
Commit: 27b3b4d470c6309729faba0b3872abf85f716f78
Parents: 9ca292a
Author: Michael Ho <kw...@cloudera.com>
Authored: Wed Jul 27 10:11:55 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Aug 10 10:07:16 2016 +0000
----------------------------------------------------------------------
be/src/benchmarks/parse-timestamp-benchmark.cc | 2 +
be/src/common/init.cc | 2 +-
be/src/exprs/CMakeLists.txt | 2 +
be/src/exprs/expr.cc | 1 +
be/src/exprs/timestamp-functions-ir.cc | 153 +----------------
be/src/exprs/timestamp-functions.cc | 176 ++++++++++++++++++++
be/src/exprs/timestamp-functions.h | 25 +--
be/src/exprs/timezone_db.cc | 6 +-
be/src/exprs/timezone_db.h | 54 ++++++
be/src/exprs/utility-functions-ir.cc | 41 +----
be/src/exprs/utility-functions.cc | 64 +++++++
be/src/exprs/utility-functions.h | 3 +
be/src/runtime/mem-tracker.cc | 1 +
be/src/runtime/timestamp-value.cc | 2 +
be/src/runtime/timestamp-value.h | 8 +-
be/src/service/impala-http-handler.cc | 1 +
be/src/statestore/statestore.cc | 1 +
be/src/util/debug-util.cc | 1 +
18 files changed, 331 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/benchmarks/parse-timestamp-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/parse-timestamp-benchmark.cc b/be/src/benchmarks/parse-timestamp-benchmark.cc
index 8824d85..68f4eac 100644
--- a/be/src/benchmarks/parse-timestamp-benchmark.cc
+++ b/be/src/benchmarks/parse-timestamp-benchmark.cc
@@ -20,11 +20,13 @@
#include <iostream>
#include <vector>
#include <sstream>
+#include <boost/date_time/posix_time/posix_time.hpp>
#include "runtime/string-value.h"
#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
#include "util/benchmark.h"
#include "util/cpu-info.h"
+
#include "common/names.h"
namespace gregorian = boost::gregorian;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 97d601b..3b49f44 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -24,7 +24,7 @@
#include "common/status.h"
#include "exec/kudu-util.h"
#include "exprs/expr.h"
-#include "exprs/timestamp-functions.h"
+#include "exprs/timezone_db.h"
#include "gutil/atomicops.h"
#include "util/cpu-info.h"
#include "util/debug-util.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 27062ff..feec660 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -50,11 +50,13 @@ add_library(Exprs
operators-ir.cc
slot-ref.cc
string-functions-ir.cc
+ timestamp-functions.cc
timestamp-functions-ir.cc
timezone_db.cc
tuple-is-null-predicate.cc
scalar-fn-call.cc
udf-builtins-ir.cc
+ utility-functions.cc
utility-functions-ir.cc
)
add_dependencies(Exprs thrift-deps gen_ir_descriptions)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/expr.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr.cc b/be/src/exprs/expr.cc
index 71758ae..0e3610d 100644
--- a/be/src/exprs/expr.cc
+++ b/be/src/exprs/expr.cc
@@ -511,6 +511,7 @@ void Expr::InitBuiltinsDummy() {
MathFunctions::Pi(NULL);
StringFunctions::Length(NULL, StringVal::null());
TimestampFunctions::Year(NULL, TimestampVal::null());
+ TimestampFunctions::UnixAndFromUnixPrepare(NULL, FunctionContext::FRAGMENT_LOCAL);
UdfBuiltins::Pi(NULL);
UtilityFunctions::Pid(NULL);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 33ace86..f250d5a 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -15,34 +15,25 @@
// specific language governing permissions and limitations
// under the License.
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <boost/date_time/gregorian/gregorian.hpp>
-#include <boost/date_time/time_zone_base.hpp>
-#include <boost/date_time/local_time/local_time.hpp>
-#include <boost/algorithm/string.hpp>
+#include "exprs/timestamp-functions.h"
+
+#include <boost/date_time/compiler_config.hpp>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
+#include <boost/date_time/gregorian/gregorian_types.hpp>
#include <ctime>
#include <gutil/strings/substitute.h>
-#include "exprs/timestamp-functions.h"
-#include "exprs/expr.h"
#include "exprs/anyval-util.h"
-
-#include "runtime/tuple-row.h"
+#include "runtime/string-value.inline.h"
#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
-#include "util/path-builder.h"
-#include "runtime/string-value.inline.h"
#include "udf/udf.h"
#include "udf/udf-internal.h"
-#include "runtime/runtime-state.h"
#include "common/names.h"
-using boost::algorithm::iequals;
using boost::gregorian::greg_month;
using boost::gregorian::min_date_time;
-using boost::local_time::local_date_time;
-using boost::local_time::time_zone_ptr;
using boost::posix_time::not_a_date_time;
using boost::posix_time::ptime;
using namespace impala_udf;
@@ -100,42 +91,6 @@ const int64_t TimestampFunctions::MAX_MILLI_INTERVAL =
const int64_t TimestampFunctions::MAX_MICRO_INTERVAL =
TimestampFunctions::MAX_MILLI_INTERVAL * 1000;
-void TimestampFunctions::UnixAndFromUnixPrepare(FunctionContext* context,
- FunctionContext::FunctionStateScope scope) {
- if (scope != FunctionContext::THREAD_LOCAL) return;
- DateTimeFormatContext* dt_ctx = NULL;
- if (context->IsArgConstant(1)) {
- StringVal fmt_val = *reinterpret_cast<StringVal*>(context->GetConstantArg(1));
- const StringValue& fmt_ref = StringValue::FromStringVal(fmt_val);
- if (fmt_val.is_null || fmt_ref.len == 0) {
- TimestampFunctions::ReportBadFormat(context, fmt_val, true);
- return;
- }
- dt_ctx = new DateTimeFormatContext(fmt_ref.ptr, fmt_ref.len);
- bool parse_result = TimestampParser::ParseFormatTokens(dt_ctx);
- if (!parse_result) {
- delete dt_ctx;
- TimestampFunctions::ReportBadFormat(context, fmt_val, true);
- return;
- }
- } else {
- // If our format string is constant, then we benefit from it only being parsed once in
- // the code above. If it's not constant, then we can reuse a context by resetting it.
- // This is much cheaper vs alloc/dealloc'ing a context for each evaluation.
- dt_ctx = new DateTimeFormatContext();
- }
- context->SetFunctionState(scope, dt_ctx);
-}
-
-void TimestampFunctions::UnixAndFromUnixClose(FunctionContext* context,
- FunctionContext::FunctionStateScope scope) {
- if (scope == FunctionContext::THREAD_LOCAL) {
- DateTimeFormatContext* dt_ctx =
- reinterpret_cast<DateTimeFormatContext*>(context->GetFunctionState(scope));
- delete dt_ctx;
- }
-}
-
StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
const TimestampValue& tv, const StringVal& fmt) {
void* state = context->GetFunctionState(FunctionContext::THREAD_LOCAL);
@@ -370,7 +325,7 @@ StringVal TimestampFunctions::ToDate(FunctionContext* context,
const TimestampVal& ts_val) {
if (ts_val.is_null) return StringVal::null();
const TimestampValue ts_value = TimestampValue::FromTimestampVal(ts_val);
- string result = to_iso_extended_string(ts_value.date());
+ string result = ToIsoExtendedString(ts_value);
return AnyValUtil::FromString(context, result);
}
@@ -793,100 +748,6 @@ IntVal TimestampFunctions::DateDiff(FunctionContext* context,
return IntVal((ts_value1.date() - ts_value2.date()).days());
}
-// This function uses inline asm functions, which we believe to be from the boost library.
-// Inline asm is not currently supported by JIT, so this function should always be run in
-// the interpreted mode. This is handled in ScalarFnCall::GetUdf().
-TimestampVal TimestampFunctions::FromUtc(FunctionContext* context,
- const TimestampVal& ts_val, const StringVal& tz_string_val) {
- if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
- const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
- if (!ts_value.HasDateOrTime()) return TimestampVal::null();
-
- const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
- time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
- string(tz_string_value.ptr, tz_string_value.len), ts_value);
- if (timezone == NULL) {
- // This should return null. Hive just ignores it.
- stringstream ss;
- ss << "Unknown timezone '" << tz_string_value << "'" << endl;
- context->AddWarning(ss.str().c_str());
- return ts_val;
- }
-
- ptime temp;
- ts_value.ToPtime(&temp);
- local_date_time lt(temp, timezone);
- TimestampValue return_value = lt.local_time();
- TimestampVal return_val;
- return_value.ToTimestampVal(&return_val);
- return return_val;
-}
-
-// This function uses inline asm functions, which we believe to be from the boost library.
-// Inline asm is not currently supported by JIT, so this function should always be run in
-// the interpreted mode. This is handled in ScalarFnCall::GetUdf().
-TimestampVal TimestampFunctions::ToUtc(FunctionContext* context,
- const TimestampVal& ts_val, const StringVal& tz_string_val) {
- if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
- const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
- if (!ts_value.HasDateOrTime()) return TimestampVal::null();
-
- const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
- time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
- string(tz_string_value.ptr, tz_string_value.len), ts_value);
- // This should raise some sort of error or at least null. Hive Just ignores it.
- if (timezone == NULL) {
- stringstream ss;
- ss << "Unknown timezone '" << tz_string_value << "'" << endl;
- context->AddWarning(ss.str().c_str());
- return ts_val;
- }
-
- local_date_time lt(ts_value.date(), ts_value.time(),
- timezone, local_date_time::NOT_DATE_TIME_ON_ERROR);
- TimestampValue return_value(lt.utc_time());
- TimestampVal return_val;
- return_value.ToTimestampVal(&return_val);
- return return_val;
-}
-
-time_zone_ptr TimezoneDatabase::FindTimezone(const string& tz, const TimestampValue& tv) {
- // The backing database does not capture some subtleties, there are special cases
- if ((tv.date().year() < 2011 || (tv.date().year() == 2011 && tv.date().month() < 4)) &&
- (iequals("Europe/Moscow", tz) || iequals("Moscow", tz) || iequals("MSK", tz))) {
- // We transition in pre April 2011 from using the tz_database_ to a custom rule
- // Russia stopped using daylight savings in 2011, the tz_database_ is
- // set up assuming Russia uses daylight saving every year.
- // Sun, Mar 27, 2:00AM Moscow clocks moved forward +1 hour (a total of GMT +4)
- // Specifically,
- // UTC Time 26 Mar 2011 22:59:59 +0000 ===> Sun Mar 27 01:59:59 MSK 2011
- // UTC Time 26 Mar 2011 23:00:00 +0000 ===> Sun Mar 27 03:00:00 MSK 2011
- // This means in 2011, The database rule will apply DST starting March 26 2011.
- // This will be a correct +4 offset, and the database rule can apply until
- // Oct 31 when tz_database_ will incorrectly attempt to turn clocks backwards 1 hour.
- return TIMEZONE_MSK_PRE_2011_DST;
- }
-
- // See if they specified a zone id
- time_zone_ptr tzp = tz_database_.time_zone_from_region(tz);
- if (tzp != NULL) return tzp;
-
- for (vector<string>::const_iterator iter = tz_region_list_.begin();
- iter != tz_region_list_.end(); ++iter) {
- time_zone_ptr tzp = tz_database_.time_zone_from_region(*iter);
- DCHECK(tzp != NULL);
- if (tzp->dst_zone_abbrev() == tz)
- return tzp;
- if (tzp->std_zone_abbrev() == tz)
- return tzp;
- if (tzp->dst_zone_name() == tz)
- return tzp;
- if (tzp->std_zone_name() == tz)
- return tzp;
- }
- return time_zone_ptr();
-}
-
// Explicit template instantiation is required for proper linking. These functions
// are only indirectly called via a function pointer provided by the opcode registry
// which does not trigger implicit template instantiation.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/timestamp-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
new file mode 100644
index 0000000..de01104
--- /dev/null
+++ b/be/src/exprs/timestamp-functions.cc
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/timestamp-functions.h"
+
+#include <boost/algorithm/string.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <ctime>
+
+#include "exprs/anyval-util.h"
+#include "exprs/timezone_db.h"
+#include "runtime/string-value.inline.h"
+#include "runtime/timestamp-parse-util.h"
+#include "runtime/timestamp-value.h"
+#include "udf/udf.h"
+#include "udf/udf-internal.h"
+
+#include "common/names.h"
+
+using boost::algorithm::iequals;
+using boost::local_time::local_date_time;
+using boost::local_time::time_zone_ptr;
+using boost::posix_time::ptime;
+using boost::posix_time::to_iso_extended_string;
+
+namespace impala {
+
+// This function is not cross-compiled to avoid including unnecessary boost library's
+// header files which bring in a bunch of unused code and global variables and increase
+// the codegen time. boost::posix_time::to_iso_extended_string() is large enough that
+// it won't benefit much from inlining.
+string TimestampFunctions::ToIsoExtendedString(const TimestampValue& ts_value) {
+ return to_iso_extended_string(ts_value.date());
+}
+
+// This function uses inline asm functions, which we believe to be from the boost library.
+// Inline asm is not currently supported by JIT, so this function should always be run in
+// the interpreted mode. This is handled in ScalarFnCall::GetUdf().
+TimestampVal TimestampFunctions::FromUtc(FunctionContext* context,
+ const TimestampVal& ts_val, const StringVal& tz_string_val) {
+ if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
+ const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (!ts_value.HasDateOrTime()) return TimestampVal::null();
+
+ const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
+ time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
+ string(tz_string_value.ptr, tz_string_value.len), ts_value);
+ if (timezone == NULL) {
+ // This should return null. Hive just ignores it.
+ stringstream ss;
+ ss << "Unknown timezone '" << tz_string_value << "'" << endl;
+ context->AddWarning(ss.str().c_str());
+ return ts_val;
+ }
+
+ ptime temp;
+ ts_value.ToPtime(&temp);
+ local_date_time lt(temp, timezone);
+ TimestampValue return_value = lt.local_time();
+ TimestampVal return_val;
+ return_value.ToTimestampVal(&return_val);
+ return return_val;
+}
+
+// This function uses inline asm functions, which we believe to be from the boost library.
+// Inline asm is not currently supported by JIT, so this function should always be run in
+// the interpreted mode. This is handled in ScalarFnCall::GetUdf().
+TimestampVal TimestampFunctions::ToUtc(FunctionContext* context,
+ const TimestampVal& ts_val, const StringVal& tz_string_val) {
+ if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
+ const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (!ts_value.HasDateOrTime()) return TimestampVal::null();
+
+ const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
+ time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
+ string(tz_string_value.ptr, tz_string_value.len), ts_value);
+ // This should raise some sort of error or at least null. Hive Just ignores it.
+ if (timezone == NULL) {
+ stringstream ss;
+ ss << "Unknown timezone '" << tz_string_value << "'" << endl;
+ context->AddWarning(ss.str().c_str());
+ return ts_val;
+ }
+
+ local_date_time lt(ts_value.date(), ts_value.time(),
+ timezone, local_date_time::NOT_DATE_TIME_ON_ERROR);
+ TimestampValue return_value(lt.utc_time());
+ TimestampVal return_val;
+ return_value.ToTimestampVal(&return_val);
+ return return_val;
+}
+
+void TimestampFunctions::UnixAndFromUnixPrepare(FunctionContext* context,
+ FunctionContext::FunctionStateScope scope) {
+ if (scope != FunctionContext::THREAD_LOCAL) return;
+ DateTimeFormatContext* dt_ctx = NULL;
+ if (context->IsArgConstant(1)) {
+ StringVal fmt_val = *reinterpret_cast<StringVal*>(context->GetConstantArg(1));
+ const StringValue& fmt_ref = StringValue::FromStringVal(fmt_val);
+ if (fmt_val.is_null || fmt_ref.len == 0) {
+ TimestampFunctions::ReportBadFormat(context, fmt_val, true);
+ return;
+ }
+ dt_ctx = new DateTimeFormatContext(fmt_ref.ptr, fmt_ref.len);
+ bool parse_result = TimestampParser::ParseFormatTokens(dt_ctx);
+ if (!parse_result) {
+ delete dt_ctx;
+ TimestampFunctions::ReportBadFormat(context, fmt_val, true);
+ return;
+ }
+ } else {
+ // If our format string is constant, then we benefit from it only being parsed once in
+ // the code above. If it's not constant, then we can reuse a context by resetting it.
+ // This is much cheaper vs alloc/dealloc'ing a context for each evaluation.
+ dt_ctx = new DateTimeFormatContext();
+ }
+ context->SetFunctionState(scope, dt_ctx);
+}
+
+void TimestampFunctions::UnixAndFromUnixClose(FunctionContext* context,
+ FunctionContext::FunctionStateScope scope) {
+ if (scope == FunctionContext::THREAD_LOCAL) {
+ DateTimeFormatContext* dt_ctx =
+ reinterpret_cast<DateTimeFormatContext*>(context->GetFunctionState(scope));
+ delete dt_ctx;
+ }
+}
+
+time_zone_ptr TimezoneDatabase::FindTimezone(const string& tz, const TimestampValue& tv) {
+ // The backing database does not capture some subtleties, there are special cases
+ if ((tv.date().year() < 2011 || (tv.date().year() == 2011 && tv.date().month() < 4)) &&
+ (iequals("Europe/Moscow", tz) || iequals("Moscow", tz) || iequals("MSK", tz))) {
+ // We transition in pre April 2011 from using the tz_database_ to a custom rule
+ // Russia stopped using daylight savings in 2011, the tz_database_ is
+ // set up assuming Russia uses daylight saving every year.
+ // Sun, Mar 27, 2:00AM Moscow clocks moved forward +1 hour (a total of GMT +4)
+ // Specifically,
+ // UTC Time 26 Mar 2011 22:59:59 +0000 ===> Sun Mar 27 01:59:59 MSK 2011
+ // UTC Time 26 Mar 2011 23:00:00 +0000 ===> Sun Mar 27 03:00:00 MSK 2011
+ // This means in 2011, The database rule will apply DST starting March 26 2011.
+ // This will be a correct +4 offset, and the database rule can apply until
+ // Oct 31 when tz_database_ will incorrectly attempt to turn clocks backwards 1 hour.
+ return TIMEZONE_MSK_PRE_2011_DST;
+ }
+
+ // See if they specified a zone id
+ time_zone_ptr tzp = tz_database_.time_zone_from_region(tz);
+ if (tzp != NULL) return tzp;
+
+ for (vector<string>::const_iterator iter = tz_region_list_.begin();
+ iter != tz_region_list_.end(); ++iter) {
+ time_zone_ptr tzp = tz_database_.time_zone_from_region(*iter);
+ DCHECK(tzp != NULL);
+ if (tzp->dst_zone_abbrev() == tz) return tzp;
+ if (tzp->std_zone_abbrev() == tz) return tzp;
+ if (tzp->dst_zone_name() == tz) return tzp;
+ if (tzp->std_zone_name() == tz) return tzp;
+ }
+ return time_zone_ptr();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/timestamp-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index 948a6b4..8abe7d7 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -19,8 +19,6 @@
#ifndef IMPALA_EXPRS_TIMESTAMP_FUNCTIONS_H
#define IMPALA_EXPRS_TIMESTAMP_FUNCTIONS_H
-#include <boost/date_time/local_time/local_time.hpp>
-
#include "common/status.h"
#include "udf/udf.h"
@@ -187,6 +185,8 @@ class TimestampFunctions {
const StringVal& format, bool is_error);
private:
+ static std::string ToIsoExtendedString(const TimestampValue& ts_value);
+
/// Static result values for DayName() function.
static const char* MONDAY;
static const char* TUESDAY;
@@ -197,27 +197,6 @@ class TimestampFunctions {
static const char* SUNDAY;
};
-/// Functions to load and access the timestamp database.
-class TimezoneDatabase {
- public:
- /// Set up the static timezone database.
- static Status Initialize();
-
- /// Converts the name of a timezone to a boost timezone object.
- /// Some countries change their timezones, the tiemstamp is required to correctly
- /// determine the timezone information.
- static boost::local_time::time_zone_ptr FindTimezone(const std::string& tz,
- const TimestampValue& tv);
-
- /// Moscow Timezone No Daylight Savings Time (GMT+4), for use after March 2011
- static const boost::local_time::time_zone_ptr TIMEZONE_MSK_PRE_2011_DST;
-
- private:
- static const char* TIMEZONE_DATABASE_STR;
- static boost::local_time::tz_database tz_database_;
- static std::vector<std::string> tz_region_list_;
-};
-
} // namespace impala
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/timezone_db.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timezone_db.cc b/be/src/exprs/timezone_db.cc
index 36d25c7..7fed7c2 100644
--- a/be/src/exprs/timezone_db.cc
+++ b/be/src/exprs/timezone_db.cc
@@ -15,8 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-#include "timestamp-functions.h"
+#include "exprs/timezone_db.h"
+#include <boost/date_time/compiler_config.hpp>
+#include <boost/date_time/local_time/posix_time_zone.hpp>
+
+#include "exprs/timestamp-functions.h"
#include "gutil/strings/substitute.h"
#include "common/names.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/timezone_db.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h
new file mode 100644
index 0000000..76d08d5
--- /dev/null
+++ b/be/src/exprs/timezone_db.h
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#ifndef IMPALA_EXPRS_TIMEZONE_DB_H
+#define IMPALA_EXPRS_TIMEZONE_DB_H
+
+#include <boost/date_time/compiler_config.hpp>
+#include <boost/date_time/local_time/local_time_types.hpp>
+#include <boost/date_time/local_time/tz_database.hpp>
+
+#include "common/status.h"
+#include "runtime/timestamp-value.h"
+
+namespace impala {
+
+/// Functions to load and access the timestamp database.
+class TimezoneDatabase {
+ public:
+ /// Set up the static timezone database.
+ static Status Initialize();
+
+ /// Converts the name of a timezone to a boost timezone object.
+ /// Some countries change their timezones, the tiemstamp is required to correctly
+ /// determine the timezone information.
+ static boost::local_time::time_zone_ptr FindTimezone(const std::string& tz,
+ const TimestampValue& tv);
+
+ /// Moscow Timezone No Daylight Savings Time (GMT+4), for use after March 2011
+ static const boost::local_time::time_zone_ptr TIMEZONE_MSK_PRE_2011_DST;
+
+ private:
+ static const char* TIMEZONE_DATABASE_STR;
+ static boost::local_time::tz_database tz_database_;
+ static std::vector<std::string> tz_region_list_;
+};
+
+} // namespace impala
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/utility-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/utility-functions-ir.cc b/be/src/exprs/utility-functions-ir.cc
index d9d361e..ab22bb7 100644
--- a/be/src/exprs/utility-functions-ir.cc
+++ b/be/src/exprs/utility-functions-ir.cc
@@ -16,11 +16,7 @@
// under the License.
#include "exprs/utility-functions.h"
-
#include <gutil/strings/substitute.h>
-#include <boost/uuid/uuid_generators.hpp>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_io.hpp>
#include "exprs/anyval-util.h"
#include "runtime/runtime-state.h"
@@ -104,6 +100,10 @@ IntVal UtilityFunctions::Pid(FunctionContext* ctx) {
return IntVal(pid);
}
+StringVal UtilityFunctions::Uuid(FunctionContext* ctx) {
+ return GenUuid(ctx);
+}
+
BooleanVal UtilityFunctions::Sleep(FunctionContext* ctx, const IntVal& milliseconds ) {
if (milliseconds.is_null) return BooleanVal::null();
SleepForMs(milliseconds.val);
@@ -117,39 +117,6 @@ StringVal UtilityFunctions::CurrentDatabase(FunctionContext* ctx) {
return (database.len > 0) ? database : StringVal::null();
}
-void UtilityFunctions::UuidPrepare(FunctionContext* ctx,
- FunctionContext::FunctionStateScope scope) {
- if (scope == FunctionContext::THREAD_LOCAL) {
- if (ctx->GetFunctionState(FunctionContext::THREAD_LOCAL) == NULL) {
- boost::uuids::random_generator* uuid_gen =
- new boost::uuids::random_generator;
- ctx->SetFunctionState(scope, uuid_gen);
- }
- }
-}
-
-StringVal UtilityFunctions::Uuid(FunctionContext* ctx) {
- void* uuid_gen = ctx->GetFunctionState(FunctionContext::THREAD_LOCAL);
- DCHECK(uuid_gen != NULL);
- boost::uuids::uuid uuid_value =
- (*reinterpret_cast<boost::uuids::random_generator*>(uuid_gen))();
- const std::string cxx_string = boost::uuids::to_string(uuid_value);
- return StringVal::CopyFrom(ctx,
- reinterpret_cast<const uint8_t*>(cxx_string.c_str()),
- cxx_string.length());
-}
-
-void UtilityFunctions::UuidClose(FunctionContext* ctx,
- FunctionContext::FunctionStateScope scope){
- if (scope == FunctionContext::THREAD_LOCAL) {
- boost::uuids::random_generator* uuid_gen =
- reinterpret_cast<boost::uuids::random_generator*>(
- ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
- DCHECK(uuid_gen != NULL);
- delete uuid_gen;
- }
-}
-
template<typename T>
StringVal UtilityFunctions::TypeOf(FunctionContext* ctx, const T& /*input_val*/) {
FunctionContext::TypeDesc type_desc = *(ctx->GetArgType(0));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/utility-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/utility-functions.cc b/be/src/exprs/utility-functions.cc
new file mode 100644
index 0000000..6159d6d
--- /dev/null
+++ b/be/src/exprs/utility-functions.cc
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/utility-functions.h"
+
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+#include "util/debug-util.h"
+
+namespace impala {
+
+void UtilityFunctions::UuidPrepare(FunctionContext* ctx,
+ FunctionContext::FunctionStateScope scope) {
+ if (scope == FunctionContext::THREAD_LOCAL) {
+ if (ctx->GetFunctionState(FunctionContext::THREAD_LOCAL) == NULL) {
+ boost::uuids::random_generator* uuid_gen =
+ new boost::uuids::random_generator;
+ ctx->SetFunctionState(scope, uuid_gen);
+ }
+ }
+}
+
+// This function is not cross-compiled to avoid including unnecessary boost library's
+// header files which bring in a bunch of unused code and global variables and increase
+// the codegen time. The indirect call in this function is expensive enough that not
+// inlining won't make much of a difference.
+StringVal UtilityFunctions::GenUuid(FunctionContext* ctx) {
+ void* uuid_gen = ctx->GetFunctionState(FunctionContext::THREAD_LOCAL);
+ DCHECK(uuid_gen != NULL);
+ boost::uuids::uuid uuid_value =
+ (*reinterpret_cast<boost::uuids::random_generator*>(uuid_gen))();
+ const std::string cxx_string = boost::uuids::to_string(uuid_value);
+ return StringVal::CopyFrom(ctx, reinterpret_cast<const uint8_t*>(cxx_string.c_str()),
+ cxx_string.length());
+}
+
+void UtilityFunctions::UuidClose(FunctionContext* ctx,
+ FunctionContext::FunctionStateScope scope){
+ if (scope == FunctionContext::THREAD_LOCAL) {
+ boost::uuids::random_generator* uuid_gen =
+ reinterpret_cast<boost::uuids::random_generator*>(
+ ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
+ DCHECK(uuid_gen != NULL);
+ delete uuid_gen;
+ }
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/exprs/utility-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/utility-functions.h b/be/src/exprs/utility-functions.h
index f8ad31d..55f8dba 100644
--- a/be/src/exprs/utility-functions.h
+++ b/be/src/exprs/utility-functions.h
@@ -72,6 +72,9 @@ class UtilityFunctions {
/// expression. input_val is not used and it is kept here in order to let
/// the compiler generate the corresponding fully-qualified function name.
template <typename T> static StringVal TypeOf(FunctionContext* ctx, const T& input_val);
+
+ private:
+ static StringVal GenUuid(FunctionContext* ctx);
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/runtime/mem-tracker.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.cc b/be/src/runtime/mem-tracker.cc
index 0f8c967..463bd5e 100644
--- a/be/src/runtime/mem-tracker.cc
+++ b/be/src/runtime/mem-tracker.cc
@@ -18,6 +18,7 @@
#include "runtime/mem-tracker.h"
#include <boost/algorithm/string/join.hpp>
+#include <boost/lexical_cast.hpp>
#include <gperftools/malloc_extension.h>
#include <gutil/strings/substitute.h>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/runtime/timestamp-value.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.cc b/be/src/runtime/timestamp-value.cc
index 25ad97f..7dfa4c6 100644
--- a/be/src/runtime/timestamp-value.cc
+++ b/be/src/runtime/timestamp-value.cc
@@ -17,6 +17,8 @@
#include "runtime/timestamp-value.h"
+#include <boost/date_time/posix_time/posix_time.hpp>
+
#include "runtime/timestamp-parse-util.h"
#include "common/names.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/runtime/timestamp-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index f458002..ab3f081 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -19,12 +19,12 @@
#ifndef IMPALA_RUNTIME_TIMESTAMP_VALUE_H
#define IMPALA_RUNTIME_TIMESTAMP_VALUE_H
+#include <boost/date_time/compiler_config.hpp>
+#include <boost/date_time/posix_time/conversion.hpp>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <ctime>
-#include <string>
-
-#include <boost/cstdint.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
#include <gflags/gflags.h>
+#include <string>
#include "udf/udf.h"
#include "util/hash-util.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/service/impala-http-handler.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-http-handler.cc b/be/src/service/impala-http-handler.cc
index 89c1a18..923c864 100644
--- a/be/src/service/impala-http-handler.cc
+++ b/be/src/service/impala-http-handler.cc
@@ -18,6 +18,7 @@
#include "service/impala-http-handler.h"
#include <sstream>
+#include <boost/lexical_cast.hpp>
#include <boost/thread/mutex.hpp>
#include <gutil/strings/substitute.h>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/statestore/statestore.cc
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index db3a566..e325292 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -17,6 +17,7 @@
#include "statestore/statestore.h"
+#include <boost/lexical_cast.hpp>
#include <boost/thread.hpp>
#include <thrift/Thrift.h>
#include <gutil/strings/substitute.h>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/27b3b4d4/be/src/util/debug-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/debug-util.cc b/be/src/util/debug-util.cc
index f82a6af..596e7d2 100644
--- a/be/src/util/debug-util.cc
+++ b/be/src/util/debug-util.cc
@@ -19,6 +19,7 @@
#include <iomanip>
#include <sstream>
+#include <boost/tokenizer.hpp>
#include "common/version.h"
#include "runtime/collection-value.h"
[2/3] incubator-impala git commit: IMPALA-3924: Ubuntu16 support
Posted by mj...@apache.org.
IMPALA-3924: Ubuntu16 support
One problem uncovered while trying to build Impala on Ubuntu16 is
that the functions 'isnan' and 'isinf' both appear in std::
(from <cmath>) and in boost::math::, but we're currently using
them without qualifiers in several places, leading to a conflict.
This patch prefaces all uses with 'std::' to disambiguate, and also
adds <cmath> imports to all files that use those functions, for
the sake of explicitness.
Another problem is that bin/make_impala.sh uses the system cmake,
which may not be compatible with the toolchain binaries. This patch
updates impala-config.sh to add the toolchain cmake to PATH, so
that we'll use it wherever we use cmake.
Change-Id: Iaa1520c1e4aa4175468ac342b14c1262fa745f7a
Reviewed-on: http://gerrit.cloudera.org:8080/3800
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9ca292a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9ca292a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9ca292a1
Branch: refs/heads/master
Commit: 9ca292a1cce53cfb81217da93f6489422dabf9b5
Parents: b66829f
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Wed Jul 27 16:32:07 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Aug 10 06:26:03 2016 +0000
----------------------------------------------------------------------
be/src/exprs/cast-functions-ir.cc | 4 +++-
be/src/exprs/udf-builtins-ir.cc | 4 ++--
be/src/runtime/raw-value-ir.cc | 18 +++++++++++-------
be/src/runtime/raw-value.inline.h | 11 ++++++-----
be/src/util/metrics-test.cc | 10 +++++-----
bin/bootstrap_toolchain.py | 5 +++--
bin/impala-config.sh | 3 ++-
7 files changed, 32 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/be/src/exprs/cast-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/cast-functions-ir.cc b/be/src/exprs/cast-functions-ir.cc
index ad8d2db..9fa8268 100644
--- a/be/src/exprs/cast-functions-ir.cc
+++ b/be/src/exprs/cast-functions-ir.cc
@@ -17,6 +17,8 @@
#include "exprs/cast-functions.h"
+#include <cmath>
+
#include <boost/lexical_cast.hpp>
#include "exprs/anyval-util.h"
@@ -127,7 +129,7 @@ CAST_TO_STRING(BigIntVal);
StringVal CastFunctions::CastToStringVal(FunctionContext* ctx, const float_type& val) { \
if (val.is_null) return StringVal::null(); \
/* val.val could be -nan, return "nan" instead */ \
- if (isnan(val.val)) return StringVal("nan"); \
+ if (std::isnan(val.val)) return StringVal("nan"); \
/* Add 1 to MAX_FLOAT_CHARS since snprintf adds a trailing '\0' */ \
StringVal sv(ctx, MAX_FLOAT_CHARS + 1); \
if (UNLIKELY(sv.is_null)) { \
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/be/src/exprs/udf-builtins-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/udf-builtins-ir.cc b/be/src/exprs/udf-builtins-ir.cc
index 73da797..376a0d0 100644
--- a/be/src/exprs/udf-builtins-ir.cc
+++ b/be/src/exprs/udf-builtins-ir.cc
@@ -95,12 +95,12 @@ BigIntVal UdfBuiltins::MinBigInt(FunctionContext* context) {
BooleanVal UdfBuiltins::IsNan(FunctionContext* context, const DoubleVal& val) {
if (val.is_null) return BooleanVal(false);
- return BooleanVal(isnan(val.val));
+ return BooleanVal(std::isnan(val.val));
}
BooleanVal UdfBuiltins::IsInf(FunctionContext* context, const DoubleVal& val) {
if (val.is_null) return BooleanVal(false);
- return BooleanVal(isinf(val.val));
+ return BooleanVal(std::isinf(val.val));
}
// The units which can be used when Truncating a Timestamp
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/be/src/runtime/raw-value-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/raw-value-ir.cc b/be/src/runtime/raw-value-ir.cc
index 5cb1fb8..c0df4c9 100644
--- a/be/src/runtime/raw-value-ir.cc
+++ b/be/src/runtime/raw-value-ir.cc
@@ -15,9 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-#include "runtime/raw-value.inline.h"
+#include "runtime/raw-value.h"
+
+#include <cmath>
#include "runtime/decimal-value.inline.h"
+#include "runtime/string-value.inline.h"
+#include "runtime/timestamp-value.h"
using namespace impala;
@@ -52,17 +56,17 @@ int RawValue::Compare(const void* v1, const void* v2, const ColumnType& type) {
// TODO: can this be faster? (just returning the difference has underflow problems)
f1 = *reinterpret_cast<const float*>(v1);
f2 = *reinterpret_cast<const float*>(v2);
- if (UNLIKELY(isnan(f1) && isnan(f2))) return 0;
- if (UNLIKELY(isnan(f1))) return -1;
- if (UNLIKELY(isnan(f2))) return 1;
+ if (UNLIKELY(std::isnan(f1) && std::isnan(f2))) return 0;
+ if (UNLIKELY(std::isnan(f1))) return -1;
+ if (UNLIKELY(std::isnan(f2))) return 1;
return f1 > f2 ? 1 : (f1 < f2 ? -1 : 0);
case TYPE_DOUBLE:
// TODO: can this be faster?
d1 = *reinterpret_cast<const double*>(v1);
d2 = *reinterpret_cast<const double*>(v2);
- if (isnan(d1) && isnan(d2)) return 0;
- if (isnan(d1)) return -1;
- if (isnan(d2)) return 1;
+ if (std::isnan(d1) && std::isnan(d2)) return 0;
+ if (std::isnan(d1)) return -1;
+ if (std::isnan(d2)) return 1;
return d1 > d2 ? 1 : (d1 < d2 ? -1 : 0);
case TYPE_STRING:
case TYPE_VARCHAR:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/be/src/runtime/raw-value.inline.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/raw-value.inline.h b/be/src/runtime/raw-value.inline.h
index 72721af..a1f1d75 100644
--- a/be/src/runtime/raw-value.inline.h
+++ b/be/src/runtime/raw-value.inline.h
@@ -21,8 +21,9 @@
#include "runtime/raw-value.h"
+#include <cmath>
+
#include <boost/functional/hash.hpp>
-#include <math.h>
#include "common/logging.h"
#include "runtime/decimal-value.inline.h"
@@ -340,12 +341,12 @@ inline void RawValue::PrintValue(const void* value, const ColumnType& type, int
float val = *reinterpret_cast<const float*>(value);
if (LIKELY(std::isfinite(val))) {
*stream << val;
- } else if (isinf(val)) {
+ } else if (std::isinf(val)) {
// 'Infinity' is Java's text representation of inf. By staying close to Java, we
// allow Hive to read text tables containing non-finite values produced by
// Impala. (The same logic applies to 'NaN', below).
*stream << (val < 0 ? "-Infinity" : "Infinity");
- } else if (isnan(val)) {
+ } else if (std::isnan(val)) {
*stream << "NaN";
}
}
@@ -355,10 +356,10 @@ inline void RawValue::PrintValue(const void* value, const ColumnType& type, int
double val = *reinterpret_cast<const double*>(value);
if (LIKELY(std::isfinite(val))) {
*stream << val;
- } else if (isinf(val)) {
+ } else if (std::isinf(val)) {
// See TYPE_FLOAT for rationale.
*stream << (val < 0 ? "-Infinity" : "Infinity");
- } else if (isnan(val)) {
+ } else if (std::isnan(val)) {
*stream << "NaN";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/be/src/util/metrics-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/metrics-test.cc b/be/src/util/metrics-test.cc
index 028d88b..9b8cbbd 100644
--- a/be/src/util/metrics-test.cc
+++ b/be/src/util/metrics-test.cc
@@ -15,16 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-#include "util/metrics.h"
-#include "util/collection-metrics.h"
-#include "util/memory-metrics.h"
-
+#include <cmath>
#include <gtest/gtest.h>
#include <boost/scoped_ptr.hpp>
#include <limits>
#include <map>
+#include "util/collection-metrics.h"
#include "util/jni-util.h"
+#include "util/memory-metrics.h"
+#include "util/metrics.h"
#include "util/thread.h"
#include "common/names.h"
@@ -130,7 +130,7 @@ TEST_F(MetricsTest, NonFiniteValues) {
AssertValue(gauge, inf, "inf");
double nan = numeric_limits<double>::quiet_NaN();
gauge->set_value(nan);
- EXPECT_TRUE(isnan(gauge->value()));
+ EXPECT_TRUE(std::isnan(gauge->value()));
EXPECT_TRUE(gauge->ToHumanReadable() == "nan");
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index f45f6a6..d8621c4 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -51,6 +51,7 @@ OS_MAPPING = {
"ubuntu14.04" : "ec2-package-ubuntu-14-04",
"ubuntu15.04" : "ec2-package-ubuntu-14-04",
"ubuntu15.10" : "ec2-package-ubuntu-14-04",
+ "ubuntu16.04" : "ec2-package-ubuntu-16-04",
}
def try_get_platform_release_label():
@@ -343,8 +344,8 @@ if __name__ == "__main__":
if not os.path.exists(toolchain_root):
os.makedirs(toolchain_root)
- packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "gcc", "gflags", "glog",
- "gperftools", "gtest", "kudu", "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4",
+ packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "gcc", "gflags",
+ "glog", "gperftools", "gtest", "kudu", "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4",
"openldap", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"]
bootstrap(toolchain_root, packages)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9ca292a1/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index d3340fa..5983e87 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -252,6 +252,7 @@ export IMPALA_BINUTILS_VERSION=2.26-p1
export IMPALA_BOOST_VERSION=1.57.0
export IMPALA_BREAKPAD_VERSION=20150612-p1
export IMPALA_BZIP2_VERSION=1.0.6-p1
+export IMPALA_CMAKE_VERSION=3.2.3-p1
export IMPALA_CYRUS_SASL_VERSION=2.1.23
export IMPALA_GCC_VERSION=4.9.2
export IMPALA_GFLAGS_VERSION=2.0
@@ -309,7 +310,7 @@ export IMPALA_AUX_WORKLOAD_DIR=$IMPALA_AUX_TEST_HOME/testdata/workloads
export IMPALA_DATASET_DIR=$IMPALA_HOME/testdata/datasets
export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets
export IMPALA_COMMON_DIR=$IMPALA_HOME/common
-export PATH=$IMPALA_HOME/bin:$PATH
+export PATH=$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH
# The directory in which all the thirdparty CDH components live.
if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then