You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/07/07 05:02:02 UTC
[2/3] incubator-impala git commit: IMPALA-3504: UDF for current
timestamp in UTC
IMPALA-3504: UDF for current timestamp in UTC
This change adds a UDF "utc_timestamp" which returns the current
date and time in UTC. Example query:
select utc_timestamp();
+-------------------------------+
| utc_timestamp() |
+-------------------------------+
| 2017-06-15 17:36:39.290773000 |
+-------------------------------+
Change-Id: I969fc805922f2bb9c8101e84f85ff2cc3b1b6729
Reviewed-on: http://gerrit.cloudera.org:8080/7203
Tested-by: Impala Public Jenkins
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9037b8e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9037b8e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9037b8e3
Branch: refs/heads/master
Commit: 9037b8e38598a9710cd2dfcf30b9fd353bdf6191
Parents: 4e17839
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Thu Jun 15 10:29:02 2017 -0700
Committer: Matthew Jacobs <mj...@cloudera.com>
Committed: Thu Jul 6 23:04:28 2017 +0000
----------------------------------------------------------------------
be/src/exprs/expr-test.cc | 31 ++++++++++++++++++++
be/src/exprs/timestamp-functions-ir.cc | 7 +++++
be/src/exprs/timestamp-functions.h | 1 +
be/src/runtime/runtime-state.cc | 3 ++
be/src/runtime/runtime-state.h | 8 +++--
be/src/runtime/timestamp-value.h | 8 +++++
be/src/service/impala-server.cc | 6 +++-
common/function-registry/impala_functions.py | 1 +
common/thrift/ImpalaInternalService.thrift | 6 +++-
.../org/apache/impala/testutil/TestUtils.java | 7 ++++-
10 files changed, 73 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index a8998ec..a1fd9a1 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -3864,6 +3864,7 @@ TEST_F(ExprTest, UtilityFunctions) {
TestStringValue("typeOf(cast(10 as DOUBLE))", "DOUBLE");
TestStringValue("typeOf(current_database())", "STRING");
TestStringValue("typeOf(now())", "TIMESTAMP");
+ TestStringValue("typeOf(utc_timestamp())", "TIMESTAMP");
TestStringValue("typeOf(cast(10 as DECIMAL))", "DECIMAL(9,0)");
TestStringValue("typeOf(0.0)", "DECIMAL(1,1)");
TestStringValue("typeOf(3.14)", "DECIMAL(3,2)");
@@ -5229,6 +5230,7 @@ TEST_F(ExprTest, TimestampFunctions) {
// Test functions with unknown expected value.
TestValidTimestampValue("now()");
+ TestValidTimestampValue("utc_timestamp()");
TestValidTimestampValue("current_timestamp()");
TestValidTimestampValue("cast(unix_timestamp() as timestamp)");
@@ -5264,6 +5266,10 @@ TEST_F(ExprTest, TimestampFunctions) {
timestamp_result = ConvertValue<TimestampValue>(GetValue("current_timestamp()",
TYPE_TIMESTAMP));
EXPECT_BETWEEN(start_time, timestamp_result, TimestampValue::LocalTime());
+ const TimestampValue utc_start_time = TimestampValue::UtcTime();
+ timestamp_result = ConvertValue<TimestampValue>(GetValue("utc_timestamp()",
+ TYPE_TIMESTAMP));
+ EXPECT_BETWEEN(utc_start_time, timestamp_result, TimestampValue::UtcTime());
// UNIX_TIMESTAMP() has second precision so the comparison start time is shifted back
// a second to ensure an earlier value.
unix_start_time =
@@ -5273,6 +5279,31 @@ TEST_F(ExprTest, TimestampFunctions) {
EXPECT_BETWEEN(TimestampValue::FromUnixTime(unix_start_time - 1), timestamp_result,
TimestampValue::LocalTime());
+ // Test that UTC and local time represent the same point in time
+ {
+ const string stmt = "select now(), utc_timestamp()";
+ vector<FieldSchema> result_types;
+ Status status = executor_->Exec(stmt, &result_types);
+ EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " << status.GetDetail();
+ DCHECK(result_types.size() == 2);
+ EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[0].type)
+ << "invalid type returned by now()";
+ EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[1].type)
+ << "invalid type returned by utc_timestamp()";
+ string result_row;
+ status = executor_->FetchResult(&result_row);
+ EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " << status.GetDetail();
+ vector<string> result_cols;
+ boost::split(result_cols, result_row, boost::is_any_of("\t"));
+ // To ensure this fails if columns are not tab separated
+ DCHECK(result_cols.size() == 2);
+ const TimestampValue local_time = ConvertValue<TimestampValue>(result_cols[0]);
+ const TimestampValue utc_timestamp = ConvertValue<TimestampValue>(result_cols[1]);
+ TimestampValue utc_converted_to_local(utc_timestamp);
+ utc_converted_to_local.UtcToLocal();
+ EXPECT_EQ(utc_converted_to_local, local_time);
+ }
+
// Test alias
TestValue("now() = current_timestamp()", TYPE_BOOLEAN, true);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 6a25ced..0cbf19e 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -304,6 +304,13 @@ TimestampVal TimestampFunctions::Now(FunctionContext* context) {
return return_val;
}
+TimestampVal TimestampFunctions::UtcTimestamp(FunctionContext* context) {
+ const TimestampValue* utc_timestamp = context->impl()->state()->utc_timestamp();
+ TimestampVal return_val;
+ utc_timestamp->ToTimestampVal(&return_val);
+ return return_val;
+}
+
// Writes 'num' as ASCII into 'dst'. If necessary, adds leading zeros to make the ASCII
// representation exactly 'len' characters. Both 'num' and 'len' must be >= 0.
static inline void IntToChar(uint8_t* dst, int num, int len) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index cbd8b98..9e447b8 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -140,6 +140,7 @@ class TimestampFunctions {
/// Date/time functions.
static TimestampVal Now(FunctionContext* context);
+ static TimestampVal UtcTimestamp(FunctionContext* context);
static StringVal ToDate(FunctionContext* context, const TimestampVal& ts_val);
static IntVal DateDiff(FunctionContext* context, const TimestampVal& ts_val1,
const TimestampVal& ts_val2);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index b05fc6b..89eec29 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -78,6 +78,8 @@ RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx& frag
fragment_ctx_(&fragment_ctx),
instance_ctx_(&instance_ctx),
now_(new TimestampValue(TimestampValue::Parse(query_state->query_ctx().now_string))),
+ utc_timestamp_(new TimestampValue(TimestampValue::Parse(
+ query_state->query_ctx().utc_timestamp_string))),
exec_env_(exec_env),
profile_(obj_pool(), "Fragment " + PrintId(instance_ctx.fragment_instance_id)),
instance_buffer_reservation_(nullptr),
@@ -93,6 +95,7 @@ RuntimeState::RuntimeState(
instance_ctx_(nullptr),
local_query_state_(query_state_),
now_(new TimestampValue(TimestampValue::Parse(qctx.now_string))),
+ utc_timestamp_(new TimestampValue(TimestampValue::Parse(qctx.utc_timestamp_string))),
exec_env_(exec_env),
profile_(obj_pool(), "<unnamed>"),
instance_buffer_reservation_(nullptr),
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 186d540..9a1d0b2 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -115,6 +115,7 @@ class RuntimeState {
return query_ctx().session.connected_user;
}
const TimestampValue* now() const { return now_.get(); }
+ const TimestampValue* utc_timestamp() const { return utc_timestamp_.get(); }
void set_now(const TimestampValue* now);
const TUniqueId& query_id() const { return query_ctx().query_id; }
const TUniqueId& fragment_instance_id() const {
@@ -338,9 +339,12 @@ class RuntimeState {
/// Provides instance id if instance_ctx_ == nullptr
TUniqueId no_instance_id_;
- /// Query-global timestamp, e.g., for implementing now(). Set from query_globals_.
- /// Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues.
+ /// Query-global timestamps for implementing now() and utc_timestamp(). Both represent
+ /// the same point in time but now_ is in local time and utc_timestamp_ is in UTC.
+ /// Set from query_globals_. Use pointer to avoid inclusion of timestampvalue.h and
+ /// avoid clang issues.
boost::scoped_ptr<TimestampValue> now_;
+ boost::scoped_ptr<TimestampValue> utc_timestamp_;
/// TODO: get rid of this and use ExecEnv::GetInstance() instead
ExecEnv* exec_env_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/timestamp-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index 83c38e9..c84eb7f 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -123,6 +123,14 @@ class TimestampValue {
return TimestampValue(boost::posix_time::microsec_clock::local_time());
}
+ /// Returns the current Coordinated Universal Time ("UTC") with microsecond accuracy.
+ /// This should not be used to time something because it is affected by adjustments to
+ /// the system clock such as a manual correction by a system admin. For timings,
+ /// use functions in util/time.h.
+ static TimestampValue UtcTime() {
+ return TimestampValue(boost::posix_time::microsec_clock::universal_time());
+ }
+
/// Returns a TimestampValue converted from a TimestampVal. The caller must ensure
/// the TimestampVal does not represent a NULL.
static TimestampValue FromTimestampVal(const impala_udf::TimestampVal& udf_value) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 37e06ac..cf91ba3 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -875,7 +875,11 @@ Status ImpalaServer::ExecuteInternal(
void ImpalaServer::PrepareQueryContext(TQueryCtx* query_ctx) {
query_ctx->__set_pid(getpid());
- query_ctx->__set_now_string(TimestampValue::LocalTime().ToString());
+ TimestampValue utc_timestamp = TimestampValue::UtcTime();
+ query_ctx->__set_utc_timestamp_string(utc_timestamp.ToString());
+ TimestampValue local_timestamp(utc_timestamp);
+ local_timestamp.UtcToLocal();
+ query_ctx->__set_now_string(local_timestamp.ToString());
query_ctx->__set_start_unix_millis(UnixMillis());
query_ctx->__set_coord_address(MakeNetworkAddress(FLAGS_hostname, FLAGS_be_port));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/function-registry/impala_functions.py
----------------------------------------------------------------------
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index b80261c..4201553 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -227,6 +227,7 @@ visible_functions = [
'_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
[['now', 'current_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions3NowEPN10impala_udf15FunctionContextE'],
+ [['utc_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions12UtcTimestampEPN10impala_udf15FunctionContextE'],
[['from_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'],
"impala::TimestampFunctions::FromUtc"],
[['to_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'],
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index c31cd7f..4aefe55 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -322,7 +322,7 @@ struct TQueryCtx {
// Session state including user.
3: required TSessionState session
- // String containing a timestamp set as the query submission time.
+ // String containing a timestamp (in local timezone) set as the query submission time.
4: required string now_string
// Process ID of the impalad to which the user is connected.
@@ -371,6 +371,10 @@ struct TQueryCtx {
// The pool to which this request has been submitted. Used to update pool statistics
// for admission control.
16: optional string request_pool
+
+ // String containing a timestamp (in UTC) set as the query submission time. It
+ // represents the same point in time as now_string
+ 17: required string utc_timestamp_string
}
// Specification of one output destination of a plan fragment
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index ce93984..5404455 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -22,8 +22,10 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
+import java.util.Date;
import java.util.Map;
import java.util.Scanner;
+import java.util.TimeZone;
import javax.json.Json;
import javax.json.JsonObject;
@@ -258,7 +260,10 @@ public class TestUtils {
queryCtx.setSession(new TSessionState(new TUniqueId(), TSessionType.BEESWAX,
defaultDb, user, new TNetworkAddress("localhost", 0)));
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSSSSS");
- queryCtx.setNow_string(formatter.format(Calendar.getInstance().getTime()));
+ Date now = Calendar.getInstance().getTime();
+ queryCtx.setNow_string(formatter.format(now));
+ formatter.setTimeZone(TimeZone.getTimeZone("GMT"));
+ queryCtx.setUtc_timestamp_string(formatter.format(now));
queryCtx.setStart_unix_millis(System.currentTimeMillis());
queryCtx.setPid(1000);
// Disable rewrites by default because some analyzer tests have non-executable