You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2018/04/10 04:48:43 UTC
[1/6] impala git commit: Fixing typo in SimpleLogger::Flush()
Repository: impala
Updated Branches:
refs/heads/master 2896b8d12 -> 65f25c767
Fixing typo in SimpleLogger::Flush()
Change-Id: I5ce0525e8db133f4b1df653b995e3eb108619050
Reviewed-on: http://gerrit.cloudera.org:8080/9893
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d1357f5e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d1357f5e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d1357f5e
Branch: refs/heads/master
Commit: d1357f5ee0dc8cbe00ee610921338e81ea7bf86d
Parents: 2896b8d
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Sat Mar 31 20:55:11 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Apr 9 21:39:19 2018 +0000
----------------------------------------------------------------------
be/src/util/simple-logger.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d1357f5e/be/src/util/simple-logger.h
----------------------------------------------------------------------
diff --git a/be/src/util/simple-logger.h b/be/src/util/simple-logger.h
index dd87722..7018ac6 100644
--- a/be/src/util/simple-logger.h
+++ b/be/src/util/simple-logger.h
@@ -45,7 +45,7 @@ class SimpleLogger {
Status AppendEntry(const std::string& entry);
/// Flushes the log file to disk by closing and re-opening the file. This function is
- /// thread safe and blocks while a WriteEntry() is in progress
+ /// thread safe and blocks while an AppendEntry() is in progress
Status Flush();
private:
[2/6] impala git commit: IMPALA-6338: Disable flaky bloom filter test
Posted by tm...@apache.org.
IMPALA-6338: Disable flaky bloom filter test
The underlying issue in IMPALA-6338 causes successful queries that are
cancelled internally due to all results having been returned to, in
rare cases, have info missing from the profile. This has caused flaky
tests but has low impact on users, and unfortunately with the current
query lifecycle logic in the coordinator, there is no simple solution.
There is ongoing work to improve query lifecycle logic in the
coordinator holistically, see IMPALA-5384. This work will eventually
address the underlying cause of IMPALA-6338. Until then, we disable
the tests that have been flaky.
Change-Id: Ie30b88fb8fb7780fc3a7153c05fdc3606145ce35
Reviewed-on: http://gerrit.cloudera.org:8080/9822
Reviewed-by: Thomas Tauber-Marshall <tm...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d437f956
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d437f956
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d437f956
Branch: refs/heads/master
Commit: d437f956cacc2afab72231b0d5ab8cc0d1d1d71e
Parents: d1357f5
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Tue Mar 27 13:09:41 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Apr 9 21:52:57 2018 +0000
----------------------------------------------------------------------
.../functional-query/queries/QueryTest/bloom_filters.test | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d437f956/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test b/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
index 593e66b..4908e20 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/bloom_filters.test
@@ -71,8 +71,9 @@ select STRAIGHT_JOIN count(*) from (select * from tpch.lineitem a LIMIT 1) a
---- RESULTS
0
---- RUNTIME_PROFILE
-row_regex: .*1 of 1 Runtime Filter Published.*
-row_regex: .*Filter 0 \(1.00 MB\).*
+# TODO: reenable once IMPALA-6338 is fixed
+#row_regex: .*1 of 1 Runtime Filter Published.*
+#row_regex: .*Filter 0 \(1.00 MB\).*
====
[3/6] impala git commit: Bump Kudu Java version.
Posted by tm...@apache.org.
Bump Kudu Java version.
For consistency, use a more updated version of the Kudu jar
in minicluster_profile=3, which now has decimal support in it.
For the =2 profile, bump up to a more recent Kudu version.
I will prepare a separate change for 2.x.
Cherry-picks: not for 2.x
Change-Id: I25f2f6c09df637d741a571ed8c21eb3612421e23
Reviewed-on: http://gerrit.cloudera.org:8080/9935
Reviewed-by: Grant Henke <gr...@apache.org>
Reviewed-by: Thomas Tauber-Marshall <tm...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2b7311af
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2b7311af
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2b7311af
Branch: refs/heads/master
Commit: 2b7311af77f676d72b566fec0bc0718bbc6d0ac1
Parents: d437f95
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Thu Apr 5 11:59:06 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Apr 9 22:10:02 2018 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/2b7311af/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 02ec950..f909887 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -168,8 +168,6 @@ fi
export IMPALA_KUDU_VERSION=0eef8e0
unset IMPALA_KUDU_URL
-# Kudu version used to identify Java client jar from maven
-export KUDU_JAVA_VERSION=1.7.0-cdh5.16.0-SNAPSHOT
# Versions of Hadoop ecosystem dependencies.
# ------------------------------------------
@@ -206,6 +204,8 @@ if [[ $IMPALA_MINICLUSTER_PROFILE_OVERRIDE == 2 ]]; then
export IMPALA_PARQUET_VERSION=1.5.0-cdh5.16.0-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
export IMPALA_KITE_VERSION=1.0.0-cdh5.16.0-SNAPSHOT
+ # Kudu version used to identify Java client jar from maven
+ export KUDU_JAVA_VERSION=1.8.0-cdh5.16.0-SNAPSHOT
elif [[ $IMPALA_MINICLUSTER_PROFILE_OVERRIDE == 3 ]]; then
export IMPALA_MINICLUSTER_PROFILE=3
@@ -218,9 +218,7 @@ elif [[ $IMPALA_MINICLUSTER_PROFILE_OVERRIDE == 3 ]]; then
export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
- # We continue using an older version of KUDU_JAVA_VERSION,
- # as it includes support for DECIMAL.
- # TODO: Jump to newer Kudu when it's published in Maven.
+ export KUDU_JAVA_VERSION=1.8.0-cdh6.x-SNAPSHOT
fi
unset IMPALA_HADOOP_URL
[4/6] impala git commit: IMPALA-6825: Fix HADOOP_CLASSPATH when using
Hadoop 3
Posted by tm...@apache.org.
IMPALA-6825: Fix HADOOP_CLASSPATH when using Hadoop 3
IMPALA-4277 added support for Hadoop 3. However, it omitted a component
from HADOOP_CLASSPATH that is required for the local minicluster to
access AWS.
I tested this manually and with this change the minicluster is able to
access HDFS.
Cherry-picks: not for 2.x
Change-Id: Ibe5784d32c6e8d04f147b26c7d2345609006ab46
Reviewed-on: http://gerrit.cloudera.org:8080/9956
Reviewed-by: Philip Zeyliger <ph...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/f76b1bf1
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/f76b1bf1
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/f76b1bf1
Branch: refs/heads/master
Commit: f76b1bf130df95ec3c4fb512a1117ffa8a04dafb
Parents: 2b7311a
Author: Lars Volker <lv...@cloudera.com>
Authored: Mon Apr 9 11:23:37 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Apr 10 00:39:47 2018 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/f76b1bf1/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f909887..0ae7713 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -512,6 +512,9 @@ if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
# Beware of adding entries from $HADOOP_HOME here, because they can change
# the order of the classpath, leading to configuration not showing up first.
HADOOP_CLASSPATH="$LZO_JAR_PATH"
+ # Add the path containing the hadoop-aws jar, which is required to access AWS from the
+ # minicluster.
+ HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${HADOOP_HOME}/share/hadoop/tools/lib/*"
fi
export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data"
[5/6] impala git commit: IMPALA-5607: part 1: breaking
extract/date_part changes
Posted by tm...@apache.org.
IMPALA-5607: part 1: breaking extract/date_part changes
This is the compatibility-breaking part of Jinchul Kim's change
to add additional units. To support nanoseconds we need to
widen the output type of these functions. We also change
the meaning of "milliseconds" to include the seconds component.
Cherry-picks: not for 2.x
Change-Id: I42d83712d9bb3a4900bec38a9c009dcf2a1fe019
Reviewed-on: http://gerrit.cloudera.org:8080/9957
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2995be82
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2995be82
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2995be82
Branch: refs/heads/master
Commit: 2995be8238898660cf68cf125c4a52dea727f898
Parents: f76b1bf
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Apr 9 11:30:10 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Apr 10 04:00:37 2018 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 124 +++++++++----------
be/src/exprs/udf-builtins-ir.cc | 42 ++++---
be/src/exprs/udf-builtins.h | 4 +-
common/function-registry/impala_functions.py | 4 +-
.../queries/QueryTest/aggregation.test | 2 +-
.../queries/QueryTest/exprs.test | 4 +-
6 files changed, 94 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index a381af9..0f3f3d4 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -6731,73 +6731,73 @@ TEST_F(ExprTest, TimestampFunctions) {
"cast(trunc(cast('2012-09-10 07:59:59' as timestamp), 'XXYYZZ') as string)");
// Extract as a regular function
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'YEAR')",
- TYPE_INT, 2006);
- TestValue("extract('2006-05-12 18:27:28.12345', 'YEAR')", TYPE_INT, 2006);
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'MoNTH')",
- TYPE_INT, 5);
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'DaY')",
- TYPE_INT, 12);
- TestValue("extract(cast('2006-05-12 06:27:28.12345' as timestamp), 'hour')",
- TYPE_INT, 6);
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'MINUTE')",
- TYPE_INT, 27);
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'SECOND')",
- TYPE_INT, 28);
- TestValue("extract(cast('2006-05-12 18:27:28.12345' as timestamp), 'MILLISECOND')",
- TYPE_INT, 123);
- TestValue("extract(cast('2006-05-13 01:27:28.12345' as timestamp), 'EPOCH')",
- TYPE_INT, 1147483648);
- TestNonOkStatus("extract(cast('2006-05-13 01:27:28.12345' as timestamp), 'foo')");
- TestNonOkStatus("extract(cast('2006-05-13 01:27:28.12345' as timestamp), NULL)");
- TestIsNull("extract(NULL, 'EPOCH')", TYPE_INT);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'YEAR')",
+ TYPE_BIGINT, 2006);
+ TestValue("extract('2006-05-12 18:27:28.123456789', 'YEAR')", TYPE_BIGINT, 2006);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'MoNTH')",
+ TYPE_BIGINT, 5);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'DaY')",
+ TYPE_BIGINT, 12);
+ TestValue("extract(cast('2006-05-12 06:27:28.123456789' as timestamp), 'hour')",
+ TYPE_BIGINT, 6);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'MINUTE')",
+ TYPE_BIGINT, 27);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'SECOND')",
+ TYPE_BIGINT, 28);
+ TestValue("extract(cast('2006-05-12 18:27:28.123456789' as timestamp), 'MILLISECOND')",
+ TYPE_BIGINT, 28123);
+ TestValue("extract(cast('2006-05-13 01:27:28.123456789' as timestamp), 'EPOCH')",
+ TYPE_BIGINT, 1147483648);
+ TestNonOkStatus("extract(cast('2006-05-13 01:27:28.123456789' as timestamp), 'foo')");
+ TestNonOkStatus("extract(cast('2006-05-13 01:27:28.123456789' as timestamp), NULL)");
+ TestIsNull("extract(NULL, 'EPOCH')", TYPE_BIGINT);
TestNonOkStatus("extract(NULL, NULL)");
// Extract using FROM keyword
- TestValue("extract(YEAR from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 2006);
- TestValue("extract(QUARTER from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 2);
- TestValue("extract(MoNTH from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 5);
- TestValue("extract(DaY from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 12);
- TestValue("extract(hour from cast('2006-05-12 06:27:28.12345' as timestamp))",
- TYPE_INT, 6);
- TestValue("extract(MINUTE from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 27);
- TestValue("extract(SECOND from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 28);
- TestValue("extract(MILLISECOND from cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 123);
- TestValue("extract(EPOCH from cast('2006-05-13 01:27:28.12345' as timestamp))",
- TYPE_INT, 1147483648);
- TestNonOkStatus("extract(foo from cast('2006-05-13 01:27:28.12345' as timestamp))");
- TestNonOkStatus("extract(NULL from cast('2006-05-13 01:27:28.12345' as timestamp))");
- TestIsNull("extract(EPOCH from NULL)", TYPE_INT);
+ TestValue("extract(YEAR from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 2006);
+ TestValue("extract(QUARTER from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 2);
+ TestValue("extract(MoNTH from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 5);
+ TestValue("extract(DaY from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 12);
+ TestValue("extract(hour from cast('2006-05-12 06:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 6);
+ TestValue("extract(MINUTE from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 27);
+ TestValue("extract(SECOND from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 28);
+ TestValue("extract(MILLISECOND from cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 28123);
+ TestValue("extract(EPOCH from cast('2006-05-13 01:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 1147483648);
+ TestNonOkStatus("extract(foo from cast('2006-05-13 01:27:28.123456789' as timestamp))");
+ TestNonOkStatus("extract(NULL from cast('2006-05-13 01:27:28.123456789' as timestamp))");
+ TestIsNull("extract(EPOCH from NULL)", TYPE_BIGINT);
// Date_part, same as extract function but with arguments swapped
- TestValue("date_part('YEAR', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 2006);
- TestValue("date_part('QUARTER', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 2);
- TestValue("date_part('MoNTH', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 5);
- TestValue("date_part('DaY', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 12);
- TestValue("date_part('hour', cast('2006-05-12 06:27:28.12345' as timestamp))",
- TYPE_INT, 6);
- TestValue("date_part('MINUTE', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 27);
- TestValue("date_part('SECOND', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 28);
- TestValue("date_part('MILLISECOND', cast('2006-05-12 18:27:28.12345' as timestamp))",
- TYPE_INT, 123);
- TestValue("date_part('EPOCH', cast('2006-05-13 01:27:28.12345' as timestamp))",
- TYPE_INT, 1147483648);
- TestNonOkStatus("date_part('foo', cast('2006-05-13 01:27:28.12345' as timestamp))");
- TestNonOkStatus("date_part(NULL, cast('2006-05-13 01:27:28.12345' as timestamp))");
- TestIsNull("date_part('EPOCH', NULL)", TYPE_INT);
+ TestValue("date_part('YEAR', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 2006);
+ TestValue("date_part('QUARTER', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 2);
+ TestValue("date_part('MoNTH', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 5);
+ TestValue("date_part('DaY', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 12);
+ TestValue("date_part('hour', cast('2006-05-12 06:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 6);
+ TestValue("date_part('MINUTE', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 27);
+ TestValue("date_part('SECOND', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 28);
+ TestValue("date_part('MILLISECOND', cast('2006-05-12 18:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 28123);
+ TestValue("date_part('EPOCH', cast('2006-05-13 01:27:28.123456789' as timestamp))",
+ TYPE_BIGINT, 1147483648);
+ TestNonOkStatus("date_part('foo', cast('2006-05-13 01:27:28.123456789' as timestamp))");
+ TestNonOkStatus("date_part(NULL, cast('2006-05-13 01:27:28.123456789' as timestamp))");
+ TestIsNull("date_part('EPOCH', NULL)", TYPE_BIGINT);
TestNonOkStatus("date_part(NULL, NULL)");
// Test with timezone offset
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/be/src/exprs/udf-builtins-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/udf-builtins-ir.cc b/be/src/exprs/udf-builtins-ir.cc
index 3c47e6b..c870fae 100644
--- a/be/src/exprs/udf-builtins-ir.cc
+++ b/be/src/exprs/udf-builtins-ir.cc
@@ -25,6 +25,7 @@
#include <string>
#include "gen-cpp/Exprs_types.h"
+#include "gutil/walltime.h"
#include "runtime/runtime-state.h"
#include "runtime/timestamp-value.h"
#include "udf/udf-internal.h"
@@ -113,6 +114,13 @@ TimestampVal UdfBuiltins::DateTrunc(
return DateTruncImpl(context, tv, unit_str);
}
+static int64_t ExtractMillisecond(const time_duration& time) {
+ // Fractional seconds are nanoseconds because Boost is configured
+ // to use nanoseconds precision
+ return time.fractional_seconds() / (NANOS_PER_MICRO * MICROS_PER_MILLI)
+ + time.seconds() * MILLIS_PER_SEC;
+}
+
// Maps the user facing name of a unit to a TExtractField
// Returns the TExtractField for the given unit
TExtractField::type StrToExtractField(FunctionContext* ctx, const StringVal& unit_str) {
@@ -130,11 +138,11 @@ TExtractField::type StrToExtractField(FunctionContext* ctx, const StringVal& uni
return TExtractField::INVALID_FIELD;
}
-IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
+BigIntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
const TimestampVal& tv) {
// resolve extract_field using the prepared state if possible, o.w. parse now
// ExtractPrepare() can only parse extract_field if user passes it as a string literal
- if (tv.is_null) return IntVal::null();
+ if (tv.is_null) return BigIntVal::null();
TExtractField::type field;
void* state = context->GetFunctionState(FunctionContext::THREAD_LOCAL);
@@ -145,7 +153,7 @@ IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
if (field == TExtractField::INVALID_FIELD) {
string string_unit(reinterpret_cast<char*>(unit_str.ptr), unit_str.len);
context->SetError(Substitute("invalid extract field: $0", string_unit).c_str());
- return IntVal::null();
+ return BigIntVal::null();
}
}
@@ -157,16 +165,16 @@ IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
case TExtractField::QUARTER:
case TExtractField::MONTH:
case TExtractField::DAY:
- if (orig_date.is_special()) return IntVal::null();
+ if (orig_date.is_special()) return BigIntVal::null();
break;
case TExtractField::HOUR:
case TExtractField::MINUTE:
case TExtractField::SECOND:
case TExtractField::MILLISECOND:
- if (time.is_special()) return IntVal::null();
+ if (time.is_special()) return BigIntVal::null();
break;
case TExtractField::EPOCH:
- if (time.is_special() || orig_date.is_special()) return IntVal::null();
+ if (time.is_special() || orig_date.is_special()) return BigIntVal::null();
break;
case TExtractField::INVALID_FIELD:
DCHECK(false);
@@ -174,44 +182,44 @@ IntVal UdfBuiltins::Extract(FunctionContext* context, const StringVal& unit_str,
switch (field) {
case TExtractField::YEAR: {
- return IntVal(orig_date.year());
+ return BigIntVal(orig_date.year());
}
case TExtractField::QUARTER: {
int m = orig_date.month();
- return IntVal((m - 1) / 3 + 1);
+ return BigIntVal((m - 1) / 3 + 1);
}
case TExtractField::MONTH: {
- return IntVal(orig_date.month());
+ return BigIntVal(orig_date.month());
}
case TExtractField::DAY: {
- return IntVal(orig_date.day());
+ return BigIntVal(orig_date.day());
}
case TExtractField::HOUR: {
- return IntVal(time.hours());
+ return BigIntVal(time.hours());
}
case TExtractField::MINUTE: {
- return IntVal(time.minutes());
+ return BigIntVal(time.minutes());
}
case TExtractField::SECOND: {
- return IntVal(time.seconds());
+ return BigIntVal(time.seconds());
}
case TExtractField::MILLISECOND: {
- return IntVal(time.total_milliseconds() - time.total_seconds() * 1000);
+ return BigIntVal(ExtractMillisecond(time));
}
case TExtractField::EPOCH: {
ptime epoch_date(date(1970, 1, 1), time_duration(0, 0, 0));
ptime cur_date(orig_date, time);
time_duration diff = cur_date - epoch_date;
- return IntVal(diff.total_seconds());
+ return BigIntVal(diff.total_seconds());
}
default: {
DCHECK(false) << field;
- return IntVal::null();
+ return BigIntVal::null();
}
}
}
-IntVal UdfBuiltins::Extract(FunctionContext* context, const TimestampVal& tv,
+BigIntVal UdfBuiltins::Extract(FunctionContext* context, const TimestampVal& tv,
const StringVal& unit_str) {
return Extract(context, unit_str, tv);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/be/src/exprs/udf-builtins.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/udf-builtins.h b/be/src/exprs/udf-builtins.h
index 4eeb33a..8aec873 100644
--- a/be/src/exprs/udf-builtins.h
+++ b/be/src/exprs/udf-builtins.h
@@ -117,12 +117,12 @@ class UdfBuiltins {
/// http://docs.oracle.com/cd/B19306_01/server.102/b14200/functions050.htm
///
/// This is used by the DATE_PART function.
- static IntVal Extract(FunctionContext* context, const StringVal& field_str,
+ static BigIntVal Extract(FunctionContext* context, const StringVal& field_str,
const TimestampVal& date);
/// This is for the EXTRACT(Timestamp, String) and EXTRACT(Timeunit FROM
/// Timestamp) functions.
- static IntVal Extract(FunctionContext* context, const TimestampVal& date,
+ static BigIntVal Extract(FunctionContext* context, const TimestampVal& date,
const StringVal& field_str);
/// This is used by the DATE_PART function.
static void ExtractPrepare(FunctionContext* context,
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/common/function-registry/impala_functions.py
----------------------------------------------------------------------
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index 6c405c9..f86098a 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -88,11 +88,11 @@ visible_functions = [
# TIMESTAMPs meaning EXTRACT(STRING, STRING) is valid. If EXTRACT(STRING, TIMESTAMP)
# is added, it takes precedence over the existing EXTRACT(TIMESTAMP, STRING)
# which could break users.
- [['extract'], 'INT', ['TIMESTAMP', 'STRING'],
+ [['extract'], 'BIGINT', ['TIMESTAMP', 'STRING'],
'_ZN6impala11UdfBuiltins7ExtractEPN10impala_udf15FunctionContextERKNS1_12TimestampValERKNS1_9StringValE',
'_ZN6impala11UdfBuiltins21SwappedExtractPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala11UdfBuiltins12ExtractCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
- [['date_part'], 'INT', ['STRING', 'TIMESTAMP'],
+ [['date_part'], 'BIGINT', ['STRING', 'TIMESTAMP'],
'_ZN6impala11UdfBuiltins7ExtractEPN10impala_udf15FunctionContextERKNS1_9StringValERKNS1_12TimestampValE',
'_ZN6impala11UdfBuiltins14ExtractPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala11UdfBuiltins12ExtractCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
index 7bb510b..88dd97c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
@@ -1231,7 +1231,7 @@ order by 1, 2;
2010,11,1350
2010,12,1395
---- TYPES
-INT,INT,BIGINT
+BIGINT,BIGINT,BIGINT
====
---- QUERY
# IMPALA-2089: Tests correct elimination of redundant predicates.
http://git-wip-us.apache.org/repos/asf/impala/blob/2995be82/testdata/workloads/functional-query/queries/QueryTest/exprs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 1ac96f0..7677aa6 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -33,7 +33,7 @@ cross join
'millisecond',10
'epoch',1361205960
---- TYPES
-string, int
+string, bigint
====
---- QUERY
# EXTRACT fields from timestamp
@@ -46,7 +46,7 @@ from alltypesagg order by id limit 5
2010,100
2010,300
---- TYPES
-int, int
+bigint, bigint
====
---- QUERY
# IS NULL predicate
[6/6] impala git commit: IMPALA-6711: loosen shell test prompt regex
Posted by tm...@apache.org.
IMPALA-6711: loosen shell test prompt regex
We have seen this test fail because the fully-qualified domain name
differed between the python test process and the impala shell process
(see JIRA for details). The exact domain name is irrelevant to the test
- we only really care about whether the prompt appeared or not.
Change-Id: I24078ef97d56e5bb32fd866af861e3a1d19c8c44
Reviewed-on: http://gerrit.cloudera.org:8080/9831
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/65f25c76
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/65f25c76
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/65f25c76
Branch: refs/heads/master
Commit: 65f25c767c9125bc7c2bd1a75269f82ab5709cfe
Parents: 2995be8
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Mar 27 15:03:04 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Apr 10 04:26:04 2018 +0000
----------------------------------------------------------------------
tests/shell/test_shell_interactive.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/65f25c76/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 17eac5b..e9049fa 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -43,6 +43,9 @@ SHELL_CMD = "%s/bin/impala-shell.sh" % os.environ['IMPALA_HOME']
SHELL_HISTORY_FILE = os.path.expanduser("~/.impalahistory")
QUERY_FILE_PATH = os.path.join(os.environ['IMPALA_HOME'], 'tests', 'shell')
+# Regex to match the interactive shell prompt that is expected after each command.
+PROMPT_REGEX = r'\[[^:]+:2100[0-9]\]'
+
class TestImpalaShellInteractive(object):
"""Test the impala shell interactively"""
@@ -226,8 +229,6 @@ class TestImpalaShellInteractive(object):
Ensure that multiline queries are preserved when they're read back from history.
Additionally, also test that comments are preserved.
"""
- # regex for pexpect, a shell prompt is expected after each command..
- prompt_regex = '\[{0}:2100[0-9]\]'.format(socket.getfqdn())
# readline gets its input from tty, so using stdin does not work.
child_proc = pexpect.spawn(SHELL_CMD)
# List of (input query, expected text in output).
@@ -240,10 +241,10 @@ class TestImpalaShellInteractive(object):
("select /*comment*/\n1;", "[4]: select /*comment*/\n1;"),
("select\n/*comm\nent*/\n1;", "[5]: select\n/*comm\nent*/\n1;")]
for query, _ in queries:
- child_proc.expect(prompt_regex)
+ child_proc.expect(PROMPT_REGEX)
child_proc.sendline(query)
child_proc.expect("Fetched 1 row\(s\) in .*s")
- child_proc.expect(prompt_regex)
+ child_proc.expect(PROMPT_REGEX)
child_proc.sendline('quit;')
p = ImpalaShell()
p.send_cmd('history')