You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/06/22 18:11:18 UTC
[6/8] impala git commit: IMPALA-3307: Add support for IANA time-zone
db
IMPALA-3307: Add support for IANA time-zone db
Impala currently uses two different libraries for timestamp
manipulations: boost and glibc.
Issues with boost:
- Time-zone database is currently hard coded in timezone_db.cc.
Impala admins cannot update it without upgrading Impala.
- Time-zone database is flat, therefore can’t track year-to-year
changes.
- Time-zone database is not updated on a regular basis.
Issues with glibc:
- Uses /usr/share/zoneinfo/ database which could be out of sync on
some of the nodes in the Impala cluster.
- Uses the host system’s local time-zone. Different nodes in the
Impala cluster might use a different local time-zone.
- Conversion functions take a global lock, which causes severe
performance degradation.
In addition to the issues above, the fact that /usr/share/zoneinfo/
and the hard-coded boost time-zone database are both in use is a
source of inconsistency in itself.
This patch makes the following changes:
- Instead of boost and glibc, impalad uses Google's CCTZ to implement
time-zone conversions.
- Introduces a new startup flag (--hdfs_zone_info_zip) to impalad to
specify an HDFS/S3/ADLS path to a zip archive that contains the
shared compiled IANA time-zone database. If the startup flag is set,
impalad will use the specified time-zone database. Otherwise,
impalad will use the default /usr/share/zoneinfo time-zone database.
- Introduces a new startup flag (--hdfs_zone_alias_conf) to impalad to
specify an HDFS/S3/ADLS path to a shared config file that contains
definitions for non-standard time-zone aliases.
- impalad reads the entire time-zone database into an in-memory
map on startup for fast lookups.
- The name of the coordinator node’s local time-zone is saved to the
query context when preparing query execution. This time-zone is used
whenever the current time-zone is referred afterwards in an
execution node.
- Adds a new ZipUtil class to extract files from a zip archive. The
implementation is not vulnerable to Zip Slip.
Cherry-picks: not for 2.x.
Change-Id: I93c1fbffe81f067919706e30db0a34d0e58e7e77
Reviewed-on: http://gerrit.cloudera.org:8080/9986
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Attila Jeges <at...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/17749dbc
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/17749dbc
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/17749dbc
Branch: refs/heads/master
Commit: 17749dbcfc51ebe67c269ce812749d1845e47e7a
Parents: a541ac6
Author: Attila Jeges <at...@cloudera.com>
Authored: Mon Nov 20 20:24:49 2017 +0100
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Jun 22 13:18:58 2018 +0000
----------------------------------------------------------------------
CMakeLists.txt | 5 +
be/CMakeLists.txt | 4 +-
be/generated-sources/gen-cpp/CMakeLists.txt | 2 +
be/src/benchmarks/CMakeLists.txt | 1 +
.../benchmarks/convert-timestamp-benchmark.cc | 833 +++++++++++++
be/src/common/global-types.h | 7 +-
be/src/common/init.cc | 2 +
be/src/exec/data-source-scan-node.cc | 8 +-
be/src/exec/data-source-scan-node.h | 6 +-
be/src/exec/hdfs-orc-scanner.cc | 2 +-
be/src/exec/parquet-column-readers.cc | 8 +-
be/src/exprs/CMakeLists.txt | 1 +
be/src/exprs/aggregate-functions-ir.cc | 6 +-
be/src/exprs/cast-functions-ir.cc | 14 +-
be/src/exprs/decimal-operators-ir.cc | 16 +-
be/src/exprs/decimal-operators.h | 8 +-
be/src/exprs/expr-test.cc | 128 +-
be/src/exprs/literal.cc | 2 -
be/src/exprs/timestamp-functions-ir.cc | 46 +-
be/src/exprs/timestamp-functions.cc | 154 +--
be/src/exprs/timezone_db-test.cc | 230 ++++
be/src/exprs/timezone_db.cc | 1134 +++++++-----------
be/src/exprs/timezone_db.h | 134 ++-
be/src/runtime/raw-value-test.cc | 5 +-
be/src/runtime/runtime-state.cc | 21 +
be/src/runtime/runtime-state.h | 5 +
be/src/runtime/timestamp-test.cc | 78 +-
be/src/runtime/timestamp-value.cc | 163 +--
be/src/runtime/timestamp-value.h | 96 +-
be/src/runtime/timestamp-value.inline.h | 48 +-
be/src/service/frontend.cc | 4 +-
be/src/service/impala-server.cc | 17 +-
be/src/service/impalad-main.cc | 8 +-
be/src/util/CMakeLists.txt | 2 +
be/src/util/filesystem-util-test.cc | 54 +
be/src/util/filesystem-util.cc | 121 ++
be/src/util/filesystem-util.h | 65 +
be/src/util/hdfs-util-test.cc | 38 +
be/src/util/hdfs-util.cc | 11 +
be/src/util/hdfs-util.h | 5 +
be/src/util/time-test.cc | 37 +-
be/src/util/time.cc | 85 +-
be/src/util/time.h | 6 +
be/src/util/zip-util-test.cc | 87 ++
be/src/util/zip-util.cc | 48 +
be/src/util/zip-util.h | 43 +
bin/bootstrap_toolchain.py | 2 +-
bin/impala-config.sh | 2 +
bin/rat_exclude_files.txt | 3 +
cmake_modules/FindCctz.cmake | 51 +
common/thrift/CMakeLists.txt | 1 +
common/thrift/ImpalaInternalService.thrift | 5 +
common/thrift/Zip.thrift | 25 +
common/thrift/metrics.json | 10 +
.../java/org/apache/impala/util/ZipUtil.java | 93 ++
.../org/apache/impala/testutil/TestUtils.java | 1 +
testdata/bin/create-load-data.sh | 3 +
testdata/data/timezoneverification.csv | 44 +-
testdata/tzdb/2017c-corrupt.zip | Bin 0 -> 208008 bytes
testdata/tzdb/2017c.zip | Bin 0 -> 486180 bytes
testdata/tzdb/alias.conf | 33 +
testdata/tzdb_tiny/America/New_York | 1 +
testdata/tzdb_tiny/Etc/GMT+4 | Bin 0 -> 120 bytes
testdata/tzdb_tiny/US/Eastern | 1 +
testdata/tzdb_tiny/UTC | 1 +
testdata/tzdb_tiny/Zulu | Bin 0 -> 118 bytes
testdata/tzdb_tiny/posix/UTC | Bin 0 -> 118 bytes
testdata/tzdb_tiny/posixrules | Bin 0 -> 3519 bytes
.../queries/QueryTest/exprs.test | 2 +-
.../test_hive_parquet_timestamp_conversion.py | 30 +-
tests/custom_cluster/test_shared_tzdb.py | 119 ++
tests/query_test/test_timezones.py | 43 -
72 files changed, 3089 insertions(+), 1179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2e31929..2b8d563 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -98,6 +98,7 @@ set_dep_root(RAPIDJSON)
set_dep_root(SNAPPY)
set_dep_root(THRIFT)
set_dep_root(ZLIB)
+set_dep_root(CCTZ)
# The boost-cmake project hasn't been maintained for years. Let's make sure we
# don't accidentally use it if it can be found.
@@ -277,6 +278,10 @@ IMPALA_ADD_THIRDPARTY_LIB(avro ${AVRO_INCLUDE_DIR} ${AVRO_STATIC_LIB} "")
find_package(Orc REQUIRED)
IMPALA_ADD_THIRDPARTY_LIB(orc ${ORC_INCLUDE_DIR} ${ORC_STATIC_LIB} "")
+# find CCTZ headers and libs
+find_package(Cctz REQUIRED)
+IMPALA_ADD_THIRDPARTY_LIB(cctz ${CCTZ_INCLUDE_DIR} ${CCTZ_STATIC_LIB} "")
+
# find protobuf headers, libs and compiler
find_package(Protobuf REQUIRED)
IMPALA_ADD_THIRDPARTY_LIB(protobuf ${PROTOBUF_INCLUDE_DIR} ${PROTOBUF_STATIC_LIBRARY}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index d817009..de94f5b 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -308,6 +308,7 @@ set(CLANG_INCLUDE_FLAGS
"-I${OPENSSL_INCLUDE_DIR}"
"-I${LDAP_INCLUDE_DIR}"
"-I${PROTOBUF_INCLUDE_DIR}"
+ "-I${CCTZ_INCLUDE_DIR}"
)
# allow linking of static libs into dynamic lib
@@ -450,7 +451,8 @@ set (IMPALA_DEPENDENCIES
avro
orc
java_jvm
- kudu_client)
+ kudu_client
+ cctz)
# Add all external dependencies. They should come after the impala libs.
set (IMPALA_LINK_LIBS ${IMPALA_LINK_LIBS}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/generated-sources/gen-cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/generated-sources/gen-cpp/CMakeLists.txt b/be/generated-sources/gen-cpp/CMakeLists.txt
index ea7735f..5052a74 100644
--- a/be/generated-sources/gen-cpp/CMakeLists.txt
+++ b/be/generated-sources/gen-cpp/CMakeLists.txt
@@ -95,6 +95,8 @@ set(SRC_FILES
Status_types.cpp
Types_constants.cpp
Types_types.cpp
+ Zip_constants.cpp
+ Zip_types.cpp
)
# keep everything in one library, the object files reference
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/benchmarks/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/CMakeLists.txt b/be/src/benchmarks/CMakeLists.txt
index 6957bd1..dfe3d23 100644
--- a/be/src/benchmarks/CMakeLists.txt
+++ b/be/src/benchmarks/CMakeLists.txt
@@ -58,5 +58,6 @@ ADD_BE_BENCHMARK(string-compare-benchmark)
ADD_BE_BENCHMARK(string-search-benchmark)
ADD_BE_BENCHMARK(thread-create-benchmark)
ADD_BE_BENCHMARK(tuple-layout-benchmark)
+ADD_BE_BENCHMARK(convert-timestamp-benchmark)
target_link_libraries(hash-benchmark Experiments)
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/benchmarks/convert-timestamp-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/convert-timestamp-benchmark.cc b/be/src/benchmarks/convert-timestamp-benchmark.cc
new file mode 100644
index 0000000..235e3d9
--- /dev/null
+++ b/be/src/benchmarks/convert-timestamp-benchmark.cc
@@ -0,0 +1,833 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <random>
+#include <sstream>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include <boost/date_time/compiler_config.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/date_time/local_time/local_time.hpp>
+
+#include <time.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "cctz/civil_time.h"
+#include "exprs/timezone_db.h"
+#include "exprs/timestamp-functions.h"
+#include "runtime/timestamp-parse-util.h"
+#include "runtime/timestamp-value.h"
+#include "runtime/timestamp-value.inline.h"
+#include "util/benchmark.h"
+#include "util/cpu-info.h"
+#include "util/pretty-printer.h"
+#include "util/stopwatch.h"
+
+#include "common/names.h"
+
+using std::random_device;
+using std::mt19937;
+using std::uniform_int_distribution;
+using std::thread;
+
+using namespace impala;
+
+// Benchmark tests for timestamp time-zone conversions
+/*
+Machine Info: Intel(R) Core(TM) i5-6600 CPU @ 3.30GHz
+
+UtcToUnixTime: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (glibc) 7.98 8.14 8.3 1X 1X 1X
+ (Google/CCTZ) 17.9 18.2 18.5 2.24X 2.24X 2.23X
+ (boost) 301 306 311 37.7X 37.5X 37.5X
+
+LocalToUnixTime: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (glibc) 0.717 0.732 0.745 1X 1X 1X
+ (Google/CCTZ) 15.3 15.5 15.8 21.3X 21.2X 21.2X
+
+FromUtc: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (boost) 1.6 1.63 1.67 1X 1X 1X
+ (Google/CCTZ) 14.5 14.8 15.2 9.06X 9.09X 9.11X
+
+ToUtc: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (boost) 1.63 1.67 1.68 1X 1X 1X
+ (Google/CCTZ) 8.7 8.9 9.05 5.34X 5.34X 5.38X
+
+UtcToLocal: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (glibc) 2.68 2.75 2.8 1X 1X 1X
+ (Google/CCTZ) 15 15.2 15.5 5.59X 5.55X 5.53X
+
+UnixTimeToLocalPtime: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (glibc) 2.69 2.75 2.8 1X 1X 1X
+ (Google/CCTZ) 14.8 15.1 15.4 5.5X 5.49X 5.52X
+
+UnixTimeToUtcPtime: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+ (relative) (relative) (relative)
+---------------------------------------------------------------------------------------------------------
+ (glibc) 18.8 19.2 19.5 1X 1X 1X
+ (Google/CCTZ) 8 8.14 8.27 0.425X 0.425X 0.424X
+ (fast path) 29.5 30.2 30.8 1.57X 1.58X 1.58X
+
+Number of threads: 8
+
+UtcToUnixTime:
+ (glibc) elapsed time: 1s020ms
+ (Google/CCTZ) elapsed time: 144ms
+ (boost) elapsed time: 10ms
+cctz speedup: 7.0784
+boost speedup: 95.1732
+
+LocalToUnixTime:
+ (glibc) elapsed time: 18s050ms
+ (Google/CCTZ) elapsed time: 212ms
+speedup: 84.9949
+
+FromUtc:
+ (boost) elapsed time: 1s519ms
+ (Google/CCTZ) elapsed time: 263ms
+speedup: 5.77003
+
+ToUtc:
+ (boost) elapsed time: 1s674ms
+ (Google/CCTZ) elapsed time: 325ms
+speedup: 5.13874
+
+UtcToLocal:
+ (glibc) elapsed time: 4s862ms
+ (Google/CCTZ) elapsed time: 263ms
+speedup: 18.4253
+
+UnixTimeToLocalPtime:
+ (glibc) elapsed time: 4s856ms
+ (Google/CCTZ) elapsed time: 259ms
+speedup: 18.7398
+
+UnixTimeToUtcPtime:
+ (glibc) elapsed time: 928ms
+ (Google/CCTZ) elapsed time: 282ms
+ (fast path) elapsed time: 90ms
+cctz speedup: 3.28187
+fast path speedup: 10.2951
+*/
+
+vector<TimestampValue> AddTestDataDateTimes(int n, const string& startstr) {
+ DateTimeFormatContext dt_ctx;
+ dt_ctx.Reset("yyyy-MMM-dd HH:mm:ss", 19);
+ TimestampParser::ParseFormatTokens(&dt_ctx);
+
+ random_device rd;
+ mt19937 gen(rd());
+ // Random days in a [0..99] days range.
+ uniform_int_distribution<int64_t> dis_days(0, 99);
+ // Random nanoseconds in a [0..15] minutes range.
+ uniform_int_distribution<int64_t> dis_nanosec(0, 900000000000L);
+
+ boost::posix_time::ptime start(boost::posix_time::time_from_string(startstr));
+ vector<TimestampValue> data;
+ for (int i = 0; i < n; ++i) {
+ start += boost::gregorian::date_duration(dis_days(gen));
+ start += boost::posix_time::nanoseconds(dis_nanosec(gen));
+ string ts = to_simple_string(start);
+ data.push_back(TimestampValue::Parse(ts.c_str(), ts.size(), dt_ctx));
+ }
+
+ return data;
+}
+
+template <class FROM, class TO, TO (*converter)(const FROM &)>
+class TestData {
+public:
+ TestData(const vector<FROM>& data) : data_(data), result_(data.size()) {}
+
+ void test_batch(int batch_size) {
+ for (int i = 0; i < batch_size; ++i) {
+ for (size_t j = 0; j < data_.size(); ++j) {
+ this->result_[j] = converter(data_[j]);
+ }
+ }
+ }
+
+ const vector<TO>& result() const { return result_; }
+
+ void add_to_benchmark(Benchmark& bm, const char* name) {
+ bm.AddBenchmark(name, run_test, this);
+ }
+
+ static void run_test(int batch_size, void *d) {
+ TestData<FROM, TO, converter>* data =
+ reinterpret_cast<TestData<FROM, TO, converter>*>(d);
+ data->test_batch(batch_size);
+ }
+
+ static uint64_t measure_multithreaded_elapsed_time(int num_of_threads, int batch_size,
+ const vector<FROM>& data, const char* label) {
+ // Create TestData for each thread.
+ vector<unique_ptr<TestData<FROM, TO, converter>>> test_data;
+ for (int i = 0; i < num_of_threads; ++i) {
+ test_data.push_back(make_unique<TestData<FROM, TO, converter>>(data));
+ }
+
+ // Create and start threads.
+ vector<unique_ptr<thread>> threads(num_of_threads);
+ StopWatch sw;
+ sw.Start();
+ for (int i = 0; i < num_of_threads; ++i) {
+ threads[i] = make_unique<thread>(
+ run_test, batch_size, test_data[i].get());
+ }
+
+ // Wait until every thread terminates.
+ for (auto& t: threads) t->join();
+ uint64_t elapsed_time = sw.ElapsedTime();
+ sw.Stop();
+
+ cout << setw(20) << label << " elapsed time: "
+ << PrettyPrinter::Print(elapsed_time, TUnit::CPU_TICKS)
+ << endl;
+
+ return elapsed_time;
+ }
+
+private:
+ const vector<FROM>& data_;
+ vector<TO> result_;
+};
+
+template <class T>
+void bail_if_results_dont_match(const vector<const vector<T>*>& test_result_vec) {
+ if (test_result_vec.size() <= 1) return;
+
+ auto b = test_result_vec.begin();
+ for (auto it = b + 1; it != test_result_vec.end(); ++it) {
+ if (**b != **it) {
+ cerr << "Results don't match.";
+ exit(1);
+ }
+ }
+}
+
+
+//
+// Test UtcToUnixTime (boost is expected to be the fastest, followed by CCTZ and glibc)
+//
+
+// CCTZ
+const Timezone* PTR_CCTZ_LOCAL_TZ = nullptr;
+
+time_t cctz_utc_to_unix_time(const TimestampValue& ts) {
+ const boost::gregorian::date& d = ts.date();
+ const boost::posix_time::time_duration& t = ts.time();
+ cctz::civil_second cs(d.year(), d.month(), d.day(), t.hours(), t.minutes(),
+ t.seconds());
+ cctz::time_point<cctz::sys_seconds> tp = cctz::convert(cs,
+ TimezoneDatabase::GetUtcTimezone());
+ cctz::sys_seconds seconds = tp.time_since_epoch();
+ return seconds.count();
+}
+
+// glibc
+time_t glibc_utc_to_unix_time(const TimestampValue& ts) {
+ const boost::posix_time::ptime temp(ts.date(), ts.time());
+ tm temp_tm = boost::posix_time::to_tm(temp);
+ return timegm(&temp_tm);
+}
+
+// boost
+time_t boost_utc_to_unix_time(const TimestampValue& ts) {
+ static const boost::gregorian::date epoch(1970,1,1);
+ return (ts.date() - epoch).days() * 24 * 60 * 60 + ts.time().total_seconds();
+}
+
+//
+// Test LocalToUnixTime (CCTZ is expected to be faster than glibc)
+//
+
+// CCTZ
+time_t cctz_local_to_unix_time(const TimestampValue& ts) {
+ const boost::gregorian::date& d = ts.date();
+ const boost::posix_time::time_duration& t = ts.time();
+ cctz::civil_second cs(d.year(), d.month(), d.day(), t.hours(), t.minutes(),
+ t.seconds());
+ cctz::time_point<cctz::sys_seconds> tp = cctz::convert(cs, *PTR_CCTZ_LOCAL_TZ);
+ cctz::sys_seconds seconds = tp.time_since_epoch();
+ return seconds.count();
+}
+
+// glibc
+time_t glibc_local_to_unix_time(const TimestampValue& ts) {
+ const boost::posix_time::ptime temp(ts.date(), ts.time());
+ tm temp_tm = boost::posix_time::to_tm(temp);
+ return mktime(&temp_tm);
+}
+
+//
+// Test UtcToLocal (CCTZ is expected to be faster than glibc)
+//
+
+// glibc
+// Constants for use with Unix times. Leap-seconds do not apply.
+const int32_t SECONDS_IN_MINUTE = 60;
+const int32_t SECONDS_IN_HOUR = 60 * SECONDS_IN_MINUTE;
+const int32_t SECONDS_IN_DAY = 24 * SECONDS_IN_HOUR;
+
+// struct tm stores month/year data as an offset
+const unsigned short TM_YEAR_OFFSET = 1900;
+const unsigned short TM_MONTH_OFFSET = 1;
+
+// Boost stores dates as an uint32_t. Since subtraction is needed, convert to signed.
+const int64_t EPOCH_DAY_NUMBER =
+ static_cast<int64_t>(
+ boost::gregorian::date(1970, boost::gregorian::Jan, 1).day_number());
+
+TimestampValue glibc_utc_to_local(const TimestampValue& ts_value) {
+ DCHECK(ts_value.HasDateAndTime());
+ try {
+ boost::gregorian::date d = ts_value.date();
+ boost::posix_time::time_duration t = ts_value.time();
+ time_t utc =
+ (static_cast<int64_t>(d.day_number()) - EPOCH_DAY_NUMBER) * SECONDS_IN_DAY +
+ t.hours() * SECONDS_IN_HOUR +
+ t.minutes() * SECONDS_IN_MINUTE +
+ t.seconds();
+ tm temp;
+ if (UNLIKELY(localtime_r(&utc, &temp) == nullptr)) {
+ return TimestampValue(boost::posix_time::ptime(boost::posix_time::not_a_date_time));
+ }
+ // Unlikely but a time zone conversion may push the value over the min/max
+ // boundary resulting in an exception.
+ d = boost::gregorian::date(
+ static_cast<unsigned short>(temp.tm_year + TM_YEAR_OFFSET),
+ static_cast<unsigned short>(temp.tm_mon + TM_MONTH_OFFSET),
+ static_cast<unsigned short>(temp.tm_mday));
+ t = boost::posix_time::time_duration(temp.tm_hour, temp.tm_min, temp.tm_sec,
+ t.fractional_seconds());
+ return TimestampValue(d, t);
+ } catch (std::exception& /* from Boost */) {
+ return TimestampValue(boost::posix_time::ptime(boost::posix_time::not_a_date_time));
+ }
+}
+
+// CCTZ
+inline cctz::time_point<cctz::sys_seconds> cctz_unix_time_to_time_point(time_t t) {
+ static const cctz::time_point<cctz::sys_seconds> epoch =
+ std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0));
+ return epoch + cctz::sys_seconds(t);
+}
+
+inline bool cctz_check_if_date_out_of_range(const cctz::civil_second& cs) {
+ return cs.year() < 1400 || cs.year() > 9999;
+}
+
+TimestampValue cctz_utc_to_local(const TimestampValue& ts_value) {
+ DCHECK(ts_value.HasDateAndTime());
+ boost::gregorian::date d;
+ boost::posix_time::time_duration t;
+ time_t unix_time;
+ if (UNLIKELY(!ts_value.UtcToUnixTime(&unix_time))) {
+ d = boost::gregorian::date(boost::gregorian::not_a_date_time);
+ t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
+ } else {
+ cctz::time_point<cctz::sys_seconds> from_tp =
+ cctz_unix_time_to_time_point(unix_time);
+ cctz::civil_second to_cs = cctz::convert(from_tp, *PTR_CCTZ_LOCAL_TZ);
+ // boost::gregorian::date() throws boost::gregorian::bad_year if year is not in the
+ // 1400..9999 range. Need to check validity before creating the date object.
+ if (UNLIKELY(cctz_check_if_date_out_of_range(to_cs))) {
+ d = boost::gregorian::date(boost::gregorian::not_a_date_time);
+ t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
+ } else {
+ d = boost::gregorian::date(to_cs.year(), to_cs.month(), to_cs.day());
+ t = boost::posix_time::time_duration(to_cs.hour(), to_cs.minute(), to_cs.second(),
+ ts_value.time().fractional_seconds());
+ }
+ }
+ return TimestampValue(d, t);
+}
+
+
+//
+// Test FromUtc (CCTZ is expected to be faster than boost)
+//
+
+// boost
+const boost::local_time::time_zone_ptr BOOST_PST_TZ(
+ new boost::local_time::posix_time_zone(string("PST-8PDT,M4.1.0,M10.1.0")));
+
+void boost_throw_if_date_out_of_range(const boost::gregorian::date& date) {
+ // Boost checks the ranges when instantiating the year/month/day representations.
+ boost::gregorian::greg_year year = date.year();
+ boost::gregorian::greg_month month = date.month();
+ boost::gregorian::greg_day day = date.day();
+ // Ensure Boost's validation is effective.
+ DCHECK_GE(year, boost::gregorian::greg_year::min());
+ DCHECK_LE(year, boost::gregorian::greg_year::max());
+ DCHECK_GE(month, boost::gregorian::greg_month::min());
+ DCHECK_LE(month, boost::gregorian::greg_month::max());
+ DCHECK_GE(day, boost::gregorian::greg_day::min());
+ DCHECK_LE(day, boost::gregorian::greg_day::max());
+}
+
+TimestampVal boost_from_utc(const TimestampVal& ts_val) {
+ if (ts_val.is_null) return TimestampVal::null();
+ const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (!ts_value.HasDateAndTime()) return TimestampVal::null();
+
+ try {
+ boost::posix_time::ptime temp;
+ ts_value.ToPtime(&temp);
+ boost::local_time::local_date_time lt(temp, BOOST_PST_TZ);
+ boost::posix_time::ptime local_time = lt.local_time();
+ boost_throw_if_date_out_of_range(local_time.date());
+ TimestampVal return_val;
+ TimestampValue(local_time).ToTimestampVal(&return_val);
+ return return_val;
+ } catch (boost::exception&) {
+ return TimestampVal::null();
+ }
+}
+
+// CCTZ
+TimestampVal cctz_from_utc(const TimestampVal& ts_val) {
+ if (ts_val.is_null) return TimestampVal::null();
+ TimestampValue ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (UNLIKELY(!ts_value.HasDateAndTime())) return TimestampVal::null();
+
+ ts_value = cctz_utc_to_local(ts_value);
+ if (UNLIKELY(!ts_value.HasDateAndTime())) return TimestampVal::null();
+
+ TimestampVal ts_val_ret;
+ ts_value.ToTimestampVal(&ts_val_ret);
+ return ts_val_ret;
+}
+
+
+//
+// Test UnixTimeToLocalPtime (CCTZ is expected to be faster than glibc)
+//
+
+// glibc
+boost::posix_time::ptime glibc_unix_time_to_local_ptime(const time_t& unix_time) {
+ tm temp_tm;
+ if (UNLIKELY(localtime_r(&unix_time, &temp_tm) == nullptr)) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ }
+ try {
+ return boost::posix_time::ptime_from_tm(temp_tm);
+ } catch (std::exception&) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ }
+}
+
+// CCTZ
+boost::posix_time::ptime cctz_unix_time_to_local_ptime(const time_t& unix_time) {
+ cctz::time_point<cctz::sys_seconds> from_tp =
+ cctz_unix_time_to_time_point(unix_time);
+ cctz::civil_second to_cs = cctz::convert(from_tp, *PTR_CCTZ_LOCAL_TZ);
+ // boost::gregorian::date() throws boost::gregorian::bad_year if year is not in the
+ // 1400..9999 range. Need to check validity before creating the date object.
+ if (UNLIKELY(cctz_check_if_date_out_of_range(to_cs))) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ } else {
+ return boost::posix_time::ptime(
+ boost::gregorian::date(to_cs.year(), to_cs.month(), to_cs.day()),
+ boost::posix_time::time_duration(to_cs.hour(), to_cs.minute(), to_cs.second()));
+ }
+}
+
+
+//
+// Test UnixTimeToUtcPtime (fast path is expected to be the fastest, followed by glibc
+// and CCTZ)
+//
+
+// glibc
+boost::posix_time::ptime glibc_unix_time_to_utc_ptime(const time_t& unix_time) {
+ tm temp_tm;
+ if (UNLIKELY(gmtime_r(&unix_time, &temp_tm) == nullptr)) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ }
+ try {
+ return boost::posix_time::ptime_from_tm(temp_tm);
+ } catch (std::exception&) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ }
+}
+
+// Fast path
+boost::posix_time::ptime fastpath_unix_time_to_utc_ptime(const time_t& unix_time) {
+ // Minimum Unix time that can be converted with from_time_t: 1677-Sep-21 00:12:44
+ const int64_t MIN_BOOST_CONVERT_UNIX_TIME = -9223372036;
+ // Maximum Unix time that can be converted with from_time_t: 2262-Apr-11 23:47:16
+ const int64_t MAX_BOOST_CONVERT_UNIX_TIME = 9223372036;
+ if (LIKELY(unix_time >= MIN_BOOST_CONVERT_UNIX_TIME &&
+ unix_time <= MAX_BOOST_CONVERT_UNIX_TIME)) {
+ try {
+ return boost::posix_time::from_time_t(unix_time);
+ } catch (std::exception&) {
+ }
+ }
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+}
+
+// CCTZ
+boost::posix_time::ptime cctz_unix_time_to_utc_ptime(const time_t& unix_time) {
+ cctz::time_point<cctz::sys_seconds> from_tp =
+ cctz_unix_time_to_time_point(unix_time);
+ cctz::civil_second to_cs = cctz::convert(from_tp, TimezoneDatabase::GetUtcTimezone());
+ // boost::gregorian::date() throws boost::gregorian::bad_year if year is not in the
+ // 1400..9999 range. Need to check validity before creating the date object.
+ if (UNLIKELY(cctz_check_if_date_out_of_range(to_cs))) {
+ return boost::posix_time::ptime(boost::posix_time::not_a_date_time);
+ } else {
+ return boost::posix_time::ptime(
+ boost::gregorian::date(to_cs.year(), to_cs.month(), to_cs.day()),
+ boost::posix_time::time_duration(to_cs.hour(), to_cs.minute(), to_cs.second()));
+ }
+}
+
+// Fast path and CCTZ implementation combined.
+boost::posix_time::ptime cctz_optimized_unix_time_to_utc_ptime(const time_t& unix_time) {
+ // Minimum Unix time that can be converted with from_time_t: 1677-Sep-21 00:12:44
+ const int64_t MIN_BOOST_CONVERT_UNIX_TIME = -9223372036;
+ // Maximum Unix time that can be converted with from_time_t: 2262-Apr-11 23:47:16
+ const int64_t MAX_BOOST_CONVERT_UNIX_TIME = 9223372036;
+ if (LIKELY(unix_time >= MIN_BOOST_CONVERT_UNIX_TIME &&
+ unix_time <= MAX_BOOST_CONVERT_UNIX_TIME)) {
+ try {
+ return boost::posix_time::from_time_t(unix_time);
+ } catch (std::exception&) {
+ }
+ }
+
+ return cctz_unix_time_to_utc_ptime(unix_time);
+}
+
+
+//
+// Test ToUtc (CCTZ is expected to be faster than boost)
+//
+
+// boost
+TimestampVal boost_to_utc(const TimestampVal& ts_val) {
+ if (ts_val.is_null) return TimestampVal::null();
+ const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (!ts_value.HasDateAndTime()) return TimestampVal::null();
+
+ try {
+ boost::local_time::local_date_time lt(ts_value.date(), ts_value.time(), BOOST_PST_TZ,
+ boost::local_time::local_date_time::NOT_DATE_TIME_ON_ERROR);
+ boost::posix_time::ptime utc_time = lt.utc_time();
+ // The utc_time() conversion does not check ranges - need to explicitly check.
+ boost_throw_if_date_out_of_range(utc_time.date());
+ TimestampVal return_val;
+ TimestampValue(utc_time).ToTimestampVal(&return_val);
+ return return_val;
+ } catch (boost::exception&) {
+ return TimestampVal::null();
+ }
+}
+
+// CCTZ
+inline time_t cctz_time_point_to_unix_time(
+ const cctz::time_point<cctz::sys_seconds>& tp) {
+ static const cctz::time_point<cctz::sys_seconds> epoch =
+ std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0));
+ return (tp - epoch).count();
+}
+
+TimestampValue cctz_local_to_utc(const TimestampValue& ts_value) {
+ DCHECK(ts_value.HasDateAndTime());
+ boost::gregorian::date d = ts_value.date();
+ boost::posix_time::time_duration t = ts_value.time();
+
+ const cctz::civil_second from_cs(d.year(), d.month(), d.day(),
+ t.hours(), t.minutes(), t.seconds());
+
+ // In case of ambiguity invalidate TimestampValue.
+ const cctz::time_zone::civil_lookup from_cl = PTR_CCTZ_LOCAL_TZ->lookup(from_cs);
+ if (UNLIKELY(from_cl.kind != cctz::time_zone::civil_lookup::UNIQUE)) {
+ d = boost::gregorian::date(boost::gregorian::not_a_date_time);
+ t = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
+ } else {
+ int64_t nanos = t.fractional_seconds();
+ boost::posix_time::ptime temp = cctz_optimized_unix_time_to_utc_ptime(
+ cctz_time_point_to_unix_time(from_cl.pre));
+
+ d = temp.date();
+ // Time-zone conversion rules don't affect fractional seconds, leave them intact.
+ t = temp.time_of_day() + boost::posix_time::nanoseconds(nanos);
+ }
+ return TimestampValue(d, t);
+}
+
+TimestampVal cctz_to_utc(const TimestampVal& ts_val) {
+ if (ts_val.is_null) return TimestampVal::null();
+ TimestampValue ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (UNLIKELY(!ts_value.HasDateAndTime())) return TimestampVal::null();
+
+ ts_value = cctz_local_to_utc(ts_value);
+ if (UNLIKELY(!ts_value.HasDateAndTime())) return TimestampVal::null();
+
+ TimestampVal ts_val_ret;
+ ts_value.ToTimestampVal(&ts_val_ret);
+ return ts_val_ret;
+}
+
+
+int main(int argc, char* argv[]) {
+ CpuInfo::Init();
+ cout << Benchmark::GetMachineInfo() << endl;
+
+ ABORT_IF_ERROR(TimezoneDatabase::Initialize());
+
+ const string& local_tz_name = TimezoneDatabase::LocalZoneName();
+ DCHECK(local_tz_name != "");
+
+ PTR_CCTZ_LOCAL_TZ = TimezoneDatabase::FindTimezone(local_tz_name);
+ DCHECK(PTR_CCTZ_LOCAL_TZ != nullptr);
+
+ TimestampParser::Init();
+
+ const vector<TimestampValue> tsvalue_data =
+ AddTestDataDateTimes(1000, "1953-04-22 01:02:03");
+
+ // Benchmark UtcToUnixTime conversion with glibc/cctz/boost
+ Benchmark bm_utc_to_unix("UtcToUnixTime");
+ TestData<TimestampValue, time_t, glibc_utc_to_unix_time> glibc_utc_to_unix_data =
+ tsvalue_data;
+ TestData<TimestampValue, time_t, cctz_utc_to_unix_time> cctz_utc_to_unix_data =
+ tsvalue_data;
+ TestData<TimestampValue, time_t, boost_utc_to_unix_time> boost_utc_to_unix_data =
+ tsvalue_data;
+
+ glibc_utc_to_unix_data.add_to_benchmark(bm_utc_to_unix, "(glibc)");
+ cctz_utc_to_unix_data.add_to_benchmark(bm_utc_to_unix, "(Google/CCTZ)");
+ boost_utc_to_unix_data.add_to_benchmark(bm_utc_to_unix, "(boost)");
+ cout << bm_utc_to_unix.Measure() << endl;
+
+ bail_if_results_dont_match(vector<const vector<time_t>*>{
+ &cctz_utc_to_unix_data.result(), &glibc_utc_to_unix_data.result(),
+ &boost_utc_to_unix_data.result()});
+
+ // Benchmark LocalToUnixTime conversion with glibc/cctz
+ Benchmark bm_local_to_unix("LocalToUnixTime");
+ TestData<TimestampValue, time_t, glibc_local_to_unix_time> glibc_local_to_unix_data =
+ tsvalue_data;
+ TestData<TimestampValue, time_t, cctz_local_to_unix_time> cctz_local_to_unix_data =
+ tsvalue_data;
+
+ glibc_local_to_unix_data.add_to_benchmark(bm_local_to_unix, "(glibc)");
+ cctz_local_to_unix_data.add_to_benchmark(bm_local_to_unix, "(Google/CCTZ)");
+ cout << bm_local_to_unix.Measure() << endl;
+
+ bail_if_results_dont_match(vector<const vector<time_t>*>{
+ &glibc_local_to_unix_data.result(), &cctz_local_to_unix_data.result()});
+
+ // Benchmark FromUtc with boost/cctz
+ vector<TimestampVal> tsval_data;
+ for (const TimestampValue& tsvalue: tsvalue_data) {
+ TimestampVal tsval;
+ tsvalue.ToTimestampVal(&tsval);
+ tsval_data.push_back(tsval);
+ }
+
+ Benchmark bm_from_utc("FromUtc");
+ TestData<TimestampVal, TimestampVal, boost_from_utc> boost_from_utc_data = tsval_data;
+ TestData<TimestampVal, TimestampVal, cctz_from_utc> cctz_from_utc_data = tsval_data;
+
+ boost_from_utc_data.add_to_benchmark(bm_from_utc, "(boost)");
+ cctz_from_utc_data.add_to_benchmark(bm_from_utc, "(Google/CCTZ)");
+ cout << bm_from_utc.Measure() << endl;
+
+ // We don't expect boost/cctz FromUtc results to match (they use non-compatible
+ // time-zone databases for conversion).
+
+ // Benchmark ToUtc with boost/cctz
+ Benchmark bm_to_utc("ToUtc");
+ TestData<TimestampVal, TimestampVal, boost_to_utc> boost_to_utc_data = tsval_data;
+ TestData<TimestampVal, TimestampVal, cctz_to_utc> cctz_to_utc_data = tsval_data;
+
+ boost_to_utc_data.add_to_benchmark(bm_to_utc, "(boost)");
+ cctz_to_utc_data.add_to_benchmark(bm_to_utc, "(Google/CCTZ)");
+ cout << bm_to_utc.Measure() << endl;
+
+ // We don't expect boost/cctz ToUtc results to match (they use non-compatible time-zone
+ // databases for conversion).
+
+ // Benchmark UtcToLocal with glibc/cctz
+ Benchmark bm_utc_to_local("UtcToLocal");
+ TestData<TimestampValue, TimestampValue, glibc_utc_to_local> glibc_utc_to_local_data =
+ tsvalue_data;
+ TestData<TimestampValue, TimestampValue, cctz_utc_to_local> cctz_utc_to_local_data =
+ tsvalue_data;
+
+ glibc_utc_to_local_data.add_to_benchmark(bm_utc_to_local, "(glibc)");
+ cctz_utc_to_local_data.add_to_benchmark(bm_utc_to_local, "(Google/CCTZ)");
+ cout << bm_utc_to_local.Measure() << endl;
+
+ bail_if_results_dont_match(vector<const vector<TimestampValue>*>{
+ &glibc_utc_to_local_data.result(), &cctz_utc_to_local_data.result()});
+
+ // Benchmark UnixTimeToLocalPtime with glibc/cctz
+ vector<time_t> time_data;
+ for (const TimestampValue& tsvalue: tsvalue_data) {
+ time_t unix_time;
+ tsvalue.ToUnixTime(TimezoneDatabase::GetUtcTimezone(), &unix_time);
+ time_data.push_back(unix_time);
+ }
+
+ Benchmark bm_unix_time_to_local_ptime("UnixTimeToLocalPtime");
+ TestData<
+ time_t,
+ boost::posix_time::ptime,
+ glibc_unix_time_to_local_ptime> glibc_unix_time_to_local_ptime_data = time_data;
+ TestData<
+ time_t,
+ boost::posix_time::ptime,
+ cctz_unix_time_to_local_ptime> cctz_unix_time_to_local_ptime_data = time_data;
+
+ glibc_unix_time_to_local_ptime_data.add_to_benchmark(bm_unix_time_to_local_ptime,
+ "(glibc)");
+ cctz_unix_time_to_local_ptime_data.add_to_benchmark(bm_unix_time_to_local_ptime,
+ "(Google/CCTZ)");
+ cout << bm_unix_time_to_local_ptime.Measure() << endl;
+
+ bail_if_results_dont_match(vector<const vector<boost::posix_time::ptime>*>{
+ &glibc_unix_time_to_local_ptime_data.result(),
+ &cctz_unix_time_to_local_ptime_data.result()});
+
+ // Benchmark UnixTimeToUtcPtime with glibc/cctz/fastpath
+ Benchmark bm_unix_time_to_utc_ptime("UnixTimeToUtcPtime");
+ TestData<
+ time_t,
+ boost::posix_time::ptime,
+ glibc_unix_time_to_utc_ptime> glibc_unix_time_to_utc_ptime_data = time_data;
+ TestData<
+ time_t,
+ boost::posix_time::ptime,
+ cctz_unix_time_to_utc_ptime> cctz_unix_time_to_utc_ptime_data = time_data;
+ TestData<
+ time_t,
+ boost::posix_time::ptime,
+ fastpath_unix_time_to_utc_ptime> fastpath_unix_time_to_utc_ptime_data = time_data;
+
+ glibc_unix_time_to_utc_ptime_data.add_to_benchmark(bm_unix_time_to_utc_ptime,
+ "(glibc)");
+ cctz_unix_time_to_utc_ptime_data.add_to_benchmark(bm_unix_time_to_utc_ptime,
+ "(Google/CCTZ)");
+ fastpath_unix_time_to_utc_ptime_data.add_to_benchmark(bm_unix_time_to_utc_ptime,
+ "(fast path)");
+ cout << bm_unix_time_to_utc_ptime.Measure() << endl;
+
+ bail_if_results_dont_match(vector<const vector<boost::posix_time::ptime>*>{
+ &glibc_unix_time_to_utc_ptime_data.result(),
+ &cctz_unix_time_to_utc_ptime_data.result(),
+ &fastpath_unix_time_to_utc_ptime_data.result()});
+
+ // If number of threads is specified, run multithreaded tests.
+ int num_of_threads = (argc < 2) ? 0 : atoi(argv[1]);
+ if (num_of_threads >= 1) {
+ const int BATCH_SIZE = 1000;
+ cout << "Number of threads: " << num_of_threads << endl;
+
+ uint64_t m1 = 0, m2 = 0, m3 = 0;
+
+ // UtcToUnixTime
+ cout << endl << "UtcToUnixTime:" << endl;
+ m1 = glibc_utc_to_unix_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(glibc)");
+ m2 = cctz_utc_to_unix_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(Google/CCTZ)");
+ m3 = boost_utc_to_unix_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(boost)");
+ cout << "cctz speedup: " << double(m1)/double(m2) << endl;
+ cout << "boost speedup: " << double(m1)/double(m3) << endl;
+
+ // LocalToUnixTime
+ cout << endl << "LocalToUnixTime:" << endl;
+ m1 = glibc_local_to_unix_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(glibc)");
+ m2 = cctz_local_to_unix_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(Google/CCTZ)");
+ cout << "speedup: " << double(m1)/double(m2) << endl;
+
+ // FromUtc
+ cout << endl << "FromUtc:" << endl;
+ m1 = boost_from_utc_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsval_data, "(boost)");
+ m2 = cctz_from_utc_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsval_data, "(Google/CCTZ)");
+ cout << "speedup: " << double(m1)/double(m2) << endl;
+
+ // ToUtc
+ cout << endl << "ToUtc:" << endl;
+ m1 = boost_to_utc_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsval_data, "(boost)");
+ m2 = cctz_to_utc_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsval_data, "(Google/CCTZ)");
+ cout << "speedup: " << double(m1)/double(m2) << endl;
+
+ // UtcToLocal
+ cout << endl << "UtcToLocal:" << endl;
+ m1 = glibc_utc_to_local_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(glibc)");
+ m2 = cctz_utc_to_local_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, tsvalue_data, "(Google/CCTZ)");
+ cout << "speedup: " << double(m1)/double(m2) << endl;
+
+ // UnixTimeToLocalPtime
+ cout << endl << "UnixTimeToLocalPtime:" << endl;
+ m1 = glibc_unix_time_to_local_ptime_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, time_data, "(glibc)");
+ m2 = cctz_unix_time_to_local_ptime_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, time_data, "(Google/CCTZ)");
+ cout << "speedup: " << double(m1)/double(m2) << endl;
+
+ // UnixTimeToUtcPtime
+ cout << endl << "UnixTimeToUtcPtime:" << endl;
+ m1 = glibc_unix_time_to_utc_ptime_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, time_data, "(glibc)");
+ m2 = cctz_unix_time_to_utc_ptime_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, time_data, "(Google/CCTZ)");
+ m3 = fastpath_unix_time_to_utc_ptime_data.measure_multithreaded_elapsed_time(
+ num_of_threads, BATCH_SIZE, time_data, "(fast path)");
+ cout << "cctz speedup: " << double(m1)/double(m2) << endl;
+ cout << "fast path speedup: " << double(m1)/double(m3) << endl;
+ }
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/common/global-types.h
----------------------------------------------------------------------
diff --git a/be/src/common/global-types.h b/be/src/common/global-types.h
index 3111e94..40a0ee1 100644
--- a/be/src/common/global-types.h
+++ b/be/src/common/global-types.h
@@ -18,8 +18,14 @@
//
/// This file contains type definitions that are used throughout the code base.
+namespace cctz {
+class time_zone;
+};
+
namespace impala {
+typedef cctz::time_zone Timezone;
+
/// for now, these are simply ints; if we find we need to generate ids in the
/// backend, we can also introduce separate classes for these to make them
/// assignment-incompatible
@@ -28,5 +34,4 @@ typedef int SlotId;
typedef int TableId;
typedef int PlanNodeId;
typedef int FragmentIdx;
-
};
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index a692a53..c689d58 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -54,6 +54,7 @@
#include "util/test-info.h"
#include "util/thread.h"
#include "util/time.h"
+#include "util/zip-util.h"
#include "common/names.h"
@@ -256,6 +257,7 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
if (init_jvm) {
ABORT_IF_ERROR(JniUtil::Init());
InitJvmLoggingSupport();
+ ZipUtil::InitJvm();
}
if (argc == -1) {
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exec/data-source-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/data-source-scan-node.cc b/be/src/exec/data-source-scan-node.cc
index a836033..d330710 100644
--- a/be/src/exec/data-source-scan-node.cc
+++ b/be/src/exec/data-source-scan-node.cc
@@ -203,7 +203,8 @@ inline Status SetDecimalVal(const ColumnType& type, char* bytes, int len,
return Status::OK();
}
-Status DataSourceScanNode::MaterializeNextRow(MemPool* tuple_pool, Tuple* tuple) {
+Status DataSourceScanNode::MaterializeNextRow(const Timezone& local_tz,
+ MemPool* tuple_pool, Tuple* tuple) {
const vector<TColumnData>& cols = input_batch_->rows.cols;
tuple->Init(tuple_desc_->byte_size());
@@ -289,7 +290,8 @@ Status DataSourceScanNode::MaterializeNextRow(MemPool* tuple_pool, Tuple* tuple)
const uint8_t* bytes = reinterpret_cast<const uint8_t*>(val.data());
*reinterpret_cast<TimestampValue*>(slot) = TimestampValue::FromUnixTimeNanos(
ReadWriteUtil::GetInt<uint64_t>(bytes),
- ReadWriteUtil::GetInt<uint32_t>(bytes + sizeof(int64_t)));
+ ReadWriteUtil::GetInt<uint32_t>(bytes + sizeof(int64_t)),
+ local_tz);
break;
}
case TYPE_DECIMAL: {
@@ -335,7 +337,7 @@ Status DataSourceScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, boo
SCOPED_TIMER(materialize_tuple_timer());
// copy rows until we hit the limit/capacity or until we exhaust input_batch_
while (!ReachedLimit() && !row_batch->AtCapacity() && InputBatchHasNext()) {
- RETURN_IF_ERROR(MaterializeNextRow(tuple_pool, tuple));
+ RETURN_IF_ERROR(MaterializeNextRow(state->local_time_zone(), tuple_pool, tuple));
int row_idx = row_batch->AddRow();
TupleRow* tuple_row = row_batch->GetRow(row_idx);
tuple_row->SetTuple(tuple_idx_, tuple);
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exec/data-source-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/data-source-scan-node.h b/be/src/exec/data-source-scan-node.h
index 0fe0b13..09c0c71 100644
--- a/be/src/exec/data-source-scan-node.h
+++ b/be/src/exec/data-source-scan-node.h
@@ -22,6 +22,7 @@
#include <string>
#include <boost/scoped_ptr.hpp>
+#include "common/global-types.h"
#include "exec/scan-node.h"
#include "exec/external-data-source-executor.h"
#include "runtime/descriptors.h"
@@ -96,8 +97,9 @@ class DataSourceScanNode : public ScanNode {
/// the next row batch.
std::vector<int> cols_next_val_idx_;
- /// Materializes the next row (next_row_idx_) into tuple.
- Status MaterializeNextRow(MemPool* mem_pool, Tuple* tuple);
+ /// Materializes the next row (next_row_idx_) into tuple. 'local_tz' is used as the
+ /// local time-zone for materializing 'TYPE_TIMESTAMP' slots.
+ Status MaterializeNextRow(const Timezone& local_tz, MemPool* mem_pool, Tuple* tuple);
/// Gets the next batch from the data source, stored in input_batch_.
Status GetNextInputBatch();
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exec/hdfs-orc-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-orc-scanner.cc b/be/src/exec/hdfs-orc-scanner.cc
index a98e16a..bbe9b12 100644
--- a/be/src/exec/hdfs-orc-scanner.cc
+++ b/be/src/exec/hdfs-orc-scanner.cc
@@ -716,7 +716,7 @@ inline Status HdfsOrcScanner::ReadRow(const orc::StructVectorBatch& batch, int r
int64_t secs = ts_batch->data.data()[row_idx];
int64_t nanos = ts_batch->nanoseconds.data()[row_idx];
*reinterpret_cast<TimestampValue*>(slot_val_ptr) =
- TimestampValue::FromUnixTimeNanos(secs, nanos);
+ TimestampValue::FromUnixTimeNanos(secs, nanos, state_->local_time_zone());
break;
}
case orc::TypeKind::DECIMAL: {
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exec/parquet-column-readers.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.cc b/be/src/exec/parquet-column-readers.cc
index f360dfb..3b13001 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -371,13 +371,17 @@ class ScalarColumnReader : public BaseScalarColumnReader {
/// The size of this column with plain encoding for FIXED_LEN_BYTE_ARRAY, or
/// the max length for VARCHAR columns. Unused otherwise.
int fixed_len_size_;
+
+ /// Query-global timezone used as local timezone when executing the query.
+ const Timezone& local_time_zone_;
};
template <typename InternalType, parquet::Type::type PARQUET_TYPE, bool MATERIALIZED>
ScalarColumnReader<InternalType, PARQUET_TYPE, MATERIALIZED>::ScalarColumnReader(
HdfsParquetScanner* parent, const SchemaNode& node, const SlotDescriptor* slot_desc)
: BaseScalarColumnReader(parent, node, slot_desc),
- dict_decoder_(parent->scan_node_->mem_tracker()) {
+ dict_decoder_(parent->scan_node_->mem_tracker()),
+ local_time_zone_(parent->state_->local_time_zone()) {
if (!MATERIALIZED) {
// We're not materializing any values, just counting them. No need (or ability) to
// initialize state used to materialize values.
@@ -680,7 +684,7 @@ bool ScalarColumnReader<TimestampValue, parquet::Type::INT96, true>::ConvertSlot
DCHECK(FLAGS_convert_legacy_hive_parquet_utc_timestamps);
TimestampValue* dst_ts = reinterpret_cast<TimestampValue*>(slot);
*dst_ts = *src;
- if (dst_ts->HasDateAndTime()) dst_ts->UtcToLocal();
+ if (dst_ts->HasDateAndTime()) dst_ts->UtcToLocal(local_time_zone_);
return true;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 14c65f5..75312b5 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -68,6 +68,7 @@ add_dependencies(Exprs gen-deps gen_ir_descriptions)
ADD_BE_LSAN_TEST(expr-test)
ADD_BE_LSAN_TEST(expr-codegen-test)
+ADD_BE_LSAN_TEST(timezone_db-test)
# expr-codegen-test includes test IR functions
COMPILE_TO_IR(expr-codegen-test.cc)
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 3bf2867..33533f5 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -331,7 +331,7 @@ void AggregateFunctions::TimestampAvgUpdate(FunctionContext* ctx,
AvgState* avg = reinterpret_cast<AvgState*>(dst->ptr);
const TimestampValue& tm_src = TimestampValue::FromTimestampVal(src);
double val;
- if (tm_src.ToSubsecondUnixTime(&val)) {
+ if (tm_src.ToSubsecondUnixTime(ctx->impl()->state()->local_time_zone(), &val)) {
avg->sum += val;
++avg->count;
}
@@ -345,7 +345,7 @@ void AggregateFunctions::TimestampAvgRemove(FunctionContext* ctx,
AvgState* avg = reinterpret_cast<AvgState*>(dst->ptr);
const TimestampValue& tm_src = TimestampValue::FromTimestampVal(src);
double val;
- if (tm_src.ToSubsecondUnixTime(&val)) {
+ if (tm_src.ToSubsecondUnixTime(ctx->impl()->state()->local_time_zone(), &val)) {
avg->sum -= val;
--avg->count;
DCHECK_GE(avg->count, 0);
@@ -357,7 +357,7 @@ TimestampVal AggregateFunctions::TimestampAvgGetValue(FunctionContext* ctx,
AvgState* val_struct = reinterpret_cast<AvgState*>(src.ptr);
if (val_struct->count == 0) return TimestampVal::null();
const TimestampValue& tv = TimestampValue::FromSubsecondUnixTime(
- val_struct->sum / val_struct->count);
+ val_struct->sum / val_struct->count, ctx->impl()->state()->local_time_zone());
if (tv.HasDate()) {
TimestampVal result;
tv.ToTimestampVal(&result);
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/cast-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/cast-functions-ir.cc b/be/src/exprs/cast-functions-ir.cc
index 26bbe9b..45f79e0 100644
--- a/be/src/exprs/cast-functions-ir.cc
+++ b/be/src/exprs/cast-functions-ir.cc
@@ -207,7 +207,9 @@ StringVal CastFunctions::CastToChar(FunctionContext* ctx, const StringVal& val)
if (val.is_null) return to_type::null(); \
TimestampValue tv = TimestampValue::FromTimestampVal(val); \
time_t result; \
- if (!tv.ToUnixTime(&result)) return to_type::null(); \
+ if (!tv.ToUnixTime(ctx->impl()->state()->local_time_zone(), &result)) { \
+ return to_type::null(); \
+ } \
return to_type(result); \
}
@@ -223,7 +225,9 @@ CAST_FROM_TIMESTAMP(BigIntVal);
if (val.is_null) return to_type::null(); \
TimestampValue tv = TimestampValue::FromTimestampVal(val); \
double result; \
- if (!tv.ToSubsecondUnixTime(&result)) return to_type::null(); \
+ if (!tv.ToSubsecondUnixTime(ctx->impl()->state()->local_time_zone(), &result)) { \
+ return to_type::null(); \
+ } \
return to_type(result);\
}
@@ -234,7 +238,8 @@ CAST_FROM_SUBSECOND_TIMESTAMP(DoubleVal);
TimestampVal CastFunctions::CastToTimestampVal(FunctionContext* ctx, \
const from_type& val) { \
if (val.is_null) return TimestampVal::null(); \
- TimestampValue timestamp_value = TimestampValue::FromSubsecondUnixTime(val.val); \
+ TimestampValue timestamp_value = TimestampValue::FromSubsecondUnixTime(val.val, \
+ ctx->impl()->state()->local_time_zone()); \
TimestampVal result; \
timestamp_value.ToTimestampVal(&result); \
return result; \
@@ -247,7 +252,8 @@ CAST_TO_SUBSECOND_TIMESTAMP(DoubleVal);
TimestampVal CastFunctions::CastToTimestampVal(FunctionContext* ctx, \
const from_type& val) { \
if (val.is_null) return TimestampVal::null(); \
- TimestampValue timestamp_value = TimestampValue::FromUnixTime(val.val); \
+ TimestampValue timestamp_value = TimestampValue::FromUnixTime(val.val, \
+ ctx->impl()->state()->local_time_zone()); \
TimestampVal result; \
timestamp_value.ToTimestampVal(&result); \
return result; \
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/decimal-operators-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/decimal-operators-ir.cc b/be/src/exprs/decimal-operators-ir.cc
index 0bde566..0a41f97 100644
--- a/be/src/exprs/decimal-operators-ir.cc
+++ b/be/src/exprs/decimal-operators-ir.cc
@@ -24,6 +24,7 @@
#include "codegen/impala-ir.h"
#include "exprs/anyval-util.h"
#include "exprs/scalar-expr.h"
+#include "exprs/timezone_db.h"
#include "runtime/decimal-value.inline.h"
#include "util/decimal-util.h"
#include "util/string-parser.h"
@@ -633,7 +634,7 @@ IR_ALWAYS_INLINE int32_t DecimalOperators::ConvertToNanoseconds(
template <typename T>
TimestampVal DecimalOperators::ConvertToTimestampVal(
- const T& decimal_value, int scale, bool round) {
+ const T& decimal_value, int scale, bool round, const Timezone& local_tz) {
typename T::StorageType seconds = decimal_value.whole_part(scale);
if (seconds < numeric_limits<int64_t>::min() ||
seconds > numeric_limits<int64_t>::max()) {
@@ -644,7 +645,8 @@ TimestampVal DecimalOperators::ConvertToTimestampVal(
ConvertToNanoseconds(decimal_value.fractional_part(scale), scale, round);
if (decimal_value.is_negative()) nanoseconds *= -1;
TimestampVal result;
- TimestampValue::FromUnixTimeNanos(seconds, nanoseconds).ToTimestampVal(&result);
+ TimestampValue::FromUnixTimeNanos(
+ seconds, nanoseconds, local_tz).ToTimestampVal(&result);
return result;
}
@@ -655,14 +657,18 @@ TimestampVal DecimalOperators::CastToTimestampVal(
int precision = ctx->impl()->GetConstFnAttr(FunctionContextImpl::ARG_TYPE_PRECISION, 0);
int scale = ctx->impl()->GetConstFnAttr(FunctionContextImpl::ARG_TYPE_SCALE, 0);
bool is_decimal_v2 = ctx->impl()->GetConstFnAttr(FunctionContextImpl::DECIMAL_V2);
+ const Timezone& local_tz = ctx->impl()->state()->local_time_zone();
TimestampVal result;
switch (ColumnType::GetDecimalByteSize(precision)) {
case 4:
- return ConvertToTimestampVal(Decimal4Value(val.val4), scale, is_decimal_v2);
+ return ConvertToTimestampVal(Decimal4Value(val.val4), scale, is_decimal_v2,
+ local_tz);
case 8:
- return ConvertToTimestampVal(Decimal8Value(val.val8), scale, is_decimal_v2);
+ return ConvertToTimestampVal(Decimal8Value(val.val8), scale, is_decimal_v2,
+ local_tz);
case 16:
- return ConvertToTimestampVal(Decimal16Value(val.val16), scale, is_decimal_v2);
+ return ConvertToTimestampVal(Decimal16Value(val.val16), scale, is_decimal_v2,
+ local_tz);
default:
DCHECK(false);
return TimestampVal::null();
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/decimal-operators.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/decimal-operators.h b/be/src/exprs/decimal-operators.h
index e34dbf1..235c1d2 100644
--- a/be/src/exprs/decimal-operators.h
+++ b/be/src/exprs/decimal-operators.h
@@ -21,6 +21,7 @@
#include <stdint.h>
+#include "common/global-types.h"
#include "runtime/decimal-value.h"
#include "udf/udf.h"
@@ -163,11 +164,12 @@ class DecimalOperators {
static T RoundDelta(const DecimalValue<T>& v, int src_scale,
int target_scale, const DecimalRoundOp& op);
- /// Converts a decimal value (interpreted as unix time) to TimestampVal. Rounds
- /// instead of truncating if 'round' is true.
+ /// Converts a decimal value (interpreted as unix time) to TimestampVal (interpreted as
+ /// local time in 'local_tz' time-zone). Rounds instead of truncating if 'round' is
+ /// true.
template <typename T>
static TimestampVal ConvertToTimestampVal(
- const T& decimal_value, int scale, bool round);
+ const T& decimal_value, int scale, bool round, const Timezone& local_tz);
/// Converts fractional 'val' with the given 'scale' to nanoseconds. Rounds
/// instead of truncating if 'round' is true.
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 29a5be4..aac0f9e 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -25,6 +25,7 @@
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/regex.hpp>
+#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>
#include "testutil/gtest-util.h"
@@ -67,6 +68,7 @@
DECLARE_bool(abort_on_config_error);
DECLARE_bool(disable_optimization_passes);
DECLARE_bool(use_utc_for_unix_timestamp_conversions);
+DECLARE_string(hdfs_zone_info_zip);
namespace posix_time = boost::posix_time;
using boost::bad_lexical_cast;
@@ -135,23 +137,30 @@ string StringToOctalLiteral(const string& s) {
// and no special permissions are needed. This is not thread-safe.
class ScopedTimeZoneOverride {
public:
- ScopedTimeZoneOverride(string time_zone) {
- original_time_zone_ = getenv("TZ");
- setenv("TZ", time_zone.c_str(), /*overwrite*/ true);
+ ScopedTimeZoneOverride(const char* tz_name)
+ : original_tz_name_(getenv("TZ")),
+ new_tz_name_(tz_name),
+ new_tz_(TimezoneDatabase::FindTimezone(new_tz_name_)) {
+ EXPECT_NE(nullptr, new_tz_) << "Could not find " << new_tz_name_ << " time zone.";
+ setenv("TZ", new_tz_name_, true);
tzset();
}
~ScopedTimeZoneOverride() {
- if (original_time_zone_ == NULL) {
+ if (original_tz_name_ == nullptr) {
unsetenv("TZ");
} else {
- setenv("TZ", original_time_zone_, /*overwrite*/ true);
+ setenv("TZ", original_tz_name_, true);
}
tzset();
}
+ const Timezone& GetTimezone() const { return *new_tz_; }
+
private:
- char* original_time_zone_;
+ const char* const original_tz_name_;
+ const char* const new_tz_name_;
+ const Timezone* new_tz_;
};
// Enable FLAGS_use_local_tz_for_unix_timestamp_conversions for the duration of the scope.
@@ -216,7 +225,8 @@ class ExprTest : public testing::Test {
default_decimal_str_ = "1.23";
default_bool_val_ = false;
default_string_val_ = "abc";
- default_timestamp_val_ = TimestampValue::FromUnixTime(1293872461);
+ default_timestamp_val_ = TimestampValue::FromUnixTime(1293872461,
+ TimezoneDatabase::GetUtcTimezone());
default_type_strs_[TYPE_TINYINT] =
lexical_cast<string>(min_int_values_[TYPE_TINYINT]);
default_type_strs_[TYPE_SMALLINT] =
@@ -1074,22 +1084,22 @@ TimestampValue ExprTest::CreateTestTimestamp(const string& val) {
template<>
TimestampValue ExprTest::CreateTestTimestamp(float val) {
- return TimestampValue::FromSubsecondUnixTime(val);
+ return TimestampValue::FromSubsecondUnixTime(val, TimezoneDatabase::GetUtcTimezone());
}
template<>
TimestampValue ExprTest::CreateTestTimestamp(double val) {
- return TimestampValue::FromSubsecondUnixTime(val);
+ return TimestampValue::FromSubsecondUnixTime(val, TimezoneDatabase::GetUtcTimezone());
}
template<>
TimestampValue ExprTest::CreateTestTimestamp(int val) {
- return TimestampValue::FromUnixTime(val);
+ return TimestampValue::FromUnixTime(val, TimezoneDatabase::GetUtcTimezone());
}
template<>
TimestampValue ExprTest::CreateTestTimestamp(int64_t val) {
- return TimestampValue::FromUnixTime(val);
+ return TimestampValue::FromUnixTime(val, TimezoneDatabase::GetUtcTimezone());
}
// Test casting 'stmt' to each of the native types. The result should be 'val'
@@ -6370,7 +6380,7 @@ TEST_F(ExprTest, TimestampFunctions) {
// These return NULL because timezone conversion makes the value out
// of range.
- TestIsNull("to_utc_timestamp(CAST(\"1400-01-01 05:00:00\" as TIMESTAMP), \"AEST\")",
+ TestIsNull("to_utc_timestamp(CAST(\"1400-01-01 05:00:00\" as TIMESTAMP), \"AET\")",
TYPE_TIMESTAMP);
TestIsNull("from_utc_timestamp(CAST(\"1400-01-01 05:00:00\" as TIMESTAMP), \"PST\")",
TYPE_TIMESTAMP);
@@ -6380,8 +6390,8 @@ TEST_F(ExprTest, TimestampFunctions) {
TYPE_TIMESTAMP);
// With support of date strings this generates a date and 0 time.
- TestStringValue(
- "cast(cast('1999-01-10' as timestamp) as string)", "1999-01-10 00:00:00");
+ TestStringValue("cast(cast('1999-01-10' as timestamp) as string)",
+ "1999-01-10 00:00:00");
// Test functions with unknown expected value.
TestValidTimestampValue("now()");
@@ -6413,33 +6423,51 @@ TEST_F(ExprTest, TimestampFunctions) {
static_cast<int64_t>(mktime(&after)));
}
- // Test that the other current time functions are also reasonable.
- TimestampValue timestamp_result;
- TimestampValue start_time = TimestampValue::FromUnixTimeMicros(UnixMicros());
- timestamp_result = ConvertValue<TimestampValue>(GetValue("now()", TYPE_TIMESTAMP));
- EXPECT_BETWEEN(start_time, timestamp_result,
- TimestampValue::FromUnixTimeMicros(UnixMicros()));
- timestamp_result = ConvertValue<TimestampValue>(GetValue("current_timestamp()",
- TYPE_TIMESTAMP));
- EXPECT_BETWEEN(start_time, timestamp_result,
- TimestampValue::FromUnixTimeMicros(UnixMicros()));
+ // Test that now() and current_timestamp() are reasonable.
+ {
+ ScopedTimeZoneOverride time_zone("PST8PDT");
+ const Timezone& local_tz = time_zone.GetTimezone();
+
+ const TimestampValue start_time =
+ TimestampValue::FromUnixTimeMicros(UnixMicros(), local_tz);
+ TimestampValue timestamp_result =
+ ConvertValue<TimestampValue>(GetValue("now()", TYPE_TIMESTAMP));
+ EXPECT_BETWEEN(start_time, timestamp_result,
+ TimestampValue::FromUnixTimeMicros(UnixMicros(), local_tz));
+ timestamp_result = ConvertValue<TimestampValue>(GetValue("current_timestamp()",
+ TYPE_TIMESTAMP));
+ EXPECT_BETWEEN(start_time, timestamp_result,
+ TimestampValue::FromUnixTimeMicros(UnixMicros(), local_tz));
+ }
+
+ // Test that utc_timestamp() is reasonable.
const TimestampValue utc_start_time =
TimestampValue::UtcFromUnixTimeMicros(UnixMicros());
- timestamp_result = ConvertValue<TimestampValue>(GetValue("utc_timestamp()",
- TYPE_TIMESTAMP));
+ TimestampValue timestamp_result =
+ ConvertValue<TimestampValue>(GetValue("utc_timestamp()", TYPE_TIMESTAMP));
EXPECT_BETWEEN(utc_start_time, timestamp_result,
TimestampValue::UtcFromUnixTimeMicros(UnixMicros()));
- // UNIX_TIMESTAMP() has second precision so the comparison start time is shifted back
- // a second to ensure an earlier value.
- unix_start_time =
- (posix_time::microsec_clock::local_time() - from_time_t(0)).total_seconds();
- timestamp_result = ConvertValue<TimestampValue>(GetValue(
- "cast(unix_timestamp() as timestamp)", TYPE_TIMESTAMP));
- EXPECT_BETWEEN(TimestampValue::FromUnixTime(unix_start_time - 1), timestamp_result,
- TimestampValue::FromUnixTimeMicros(UnixMicros()));
+
+ // Test cast(unix_timestamp() as timestamp).
+ {
+ ScopedTimeZoneOverride time_zone("PST8PDT");
+ const Timezone& local_tz = time_zone.GetTimezone();
+
+ // UNIX_TIMESTAMP() has second precision so the comparison start time is shifted back
+ // a second to ensure an earlier value.
+ unix_start_time =
+ (posix_time::microsec_clock::local_time() - from_time_t(0)).total_seconds();
+ timestamp_result = ConvertValue<TimestampValue>(GetValue(
+ "cast(unix_timestamp() as timestamp)", TYPE_TIMESTAMP));
+ EXPECT_BETWEEN(TimestampValue::FromUnixTime(unix_start_time - 1, local_tz),
+ timestamp_result,
+ TimestampValue::FromUnixTimeMicros(UnixMicros(), local_tz));
+ }
// Test that UTC and local time represent the same point in time
{
+ ScopedTimeZoneOverride time_zone_override("PST8PDT");
+
const string stmt = "select now(), utc_timestamp()";
vector<FieldSchema> result_types;
Status status = executor_->Exec(stmt, &result_types);
@@ -6458,8 +6486,9 @@ TEST_F(ExprTest, TimestampFunctions) {
DCHECK(result_cols.size() == 2);
const TimestampValue local_time = ConvertValue<TimestampValue>(result_cols[0]);
const TimestampValue utc_timestamp = ConvertValue<TimestampValue>(result_cols[1]);
+
TimestampValue utc_converted_to_local(utc_timestamp);
- utc_converted_to_local.UtcToLocal();
+ utc_converted_to_local.UtcToLocal(time_zone_override.GetTimezone());
EXPECT_EQ(utc_converted_to_local, local_time);
}
@@ -6982,8 +7011,10 @@ TEST_F(ExprTest, ConditionalFunctions) {
TestValue("if(FALSE, cast(5.5 as double), cast(8.8 as double))", TYPE_DOUBLE, 8.8);
TestStringValue("if(TRUE, 'abc', 'defgh')", "abc");
TestStringValue("if(FALSE, 'abc', 'defgh')", "defgh");
- TimestampValue then_val = TimestampValue::FromUnixTime(1293872461);
- TimestampValue else_val = TimestampValue::FromUnixTime(929387245);
+ TimestampValue then_val = TimestampValue::FromUnixTime(1293872461,
+ TimezoneDatabase::GetUtcTimezone());
+ TimestampValue else_val = TimestampValue::FromUnixTime(929387245,
+ TimezoneDatabase::GetUtcTimezone());
TestTimestampValue("if(TRUE, cast('2011-01-01 09:01:01' as timestamp), "
"cast('1999-06-14 19:07:25' as timestamp))", then_val);
TestTimestampValue("if(FALSE, cast('2011-01-01 09:01:01' as timestamp), "
@@ -7000,8 +7031,10 @@ TEST_F(ExprTest, ConditionalFunctions) {
TestValue("nvl2(NULL, cast(5.5 as double), cast(8.8 as double))", TYPE_DOUBLE, 8.8);
TestStringValue("nvl2('some string', 'abc', 'defgh')", "abc");
TestStringValue("nvl2(NULL, 'abc', 'defgh')", "defgh");
- TimestampValue first_val = TimestampValue::FromUnixTime(1293872461);
- TimestampValue second_val = TimestampValue::FromUnixTime(929387245);
+ TimestampValue first_val = TimestampValue::FromUnixTime(1293872461,
+ TimezoneDatabase::GetUtcTimezone());
+ TimestampValue second_val = TimestampValue::FromUnixTime(929387245,
+ TimezoneDatabase::GetUtcTimezone());
TestTimestampValue("nvl2(FALSE, cast('2011-01-01 09:01:01' as timestamp), "
"cast('1999-06-14 19:07:25' as timestamp))", first_val);
TestTimestampValue("nvl2(NULL, cast('2011-01-01 09:01:01' as timestamp), "
@@ -7027,7 +7060,8 @@ TEST_F(ExprTest, ConditionalFunctions) {
TestStringValue("nullif('abc', NULL)", "abc");
TestIsNull("nullif(cast('2011-01-01 09:01:01' as timestamp), "
"cast('2011-01-01 09:01:01' as timestamp))", TYPE_TIMESTAMP);
- TimestampValue testlhs = TimestampValue::FromUnixTime(1293872461);
+ TimestampValue testlhs = TimestampValue::FromUnixTime(1293872461,
+ TimezoneDatabase::GetUtcTimezone());
TestTimestampValue("nullif(cast('2011-01-01 09:01:01' as timestamp), "
"cast('1999-06-14 19:07:25' as timestamp))", testlhs);
TestIsNull("nullif(NULL, "
@@ -7090,8 +7124,10 @@ TEST_F(ExprTest, ConditionalFunctions) {
TestStringValue("coalesce(NULL, 'abc', NULL)", "abc");
TestStringValue("coalesce('defgh', NULL, 'abc', NULL)", "defgh");
TestStringValue("coalesce(NULL, NULL, NULL, 'abc', NULL, NULL)", "abc");
- TimestampValue ats = TimestampValue::FromUnixTime(1293872461);
- TimestampValue bts = TimestampValue::FromUnixTime(929387245);
+ TimestampValue ats = TimestampValue::FromUnixTime(1293872461,
+ TimezoneDatabase::GetUtcTimezone());
+ TimestampValue bts = TimestampValue::FromUnixTime(929387245,
+ TimezoneDatabase::GetUtcTimezone());
TestTimestampValue("coalesce(cast('2011-01-01 09:01:01' as timestamp))", ats);
TestTimestampValue("coalesce(NULL, cast('2011-01-01 09:01:01' as timestamp),"
"NULL)", ats);
@@ -8671,10 +8707,16 @@ TEST_F(ExprTest, DateTruncTest) {
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
InitCommonRuntime(argc, argv, true, TestInfo::BE_TEST);
- ABORT_IF_ERROR(TimezoneDatabase::Initialize());
InitFeSupport(false);
ABORT_IF_ERROR(impala::LlvmCodeGen::InitializeLlvm());
+ // The host running this test might have an out-of-date tzdata package installed.
+ // To avoid tzdata related issues, we will load time-zone db from the testdata
+ // directory.
+ FLAGS_hdfs_zone_info_zip = Substitute("file://$0/testdata/tzdb/2017c.zip",
+ getenv("IMPALA_HOME"));
+ ABORT_IF_ERROR(TimezoneDatabase::Initialize());
+
// Disable llvm optimization passes if the env var is no set to true. Running without
// the optimizations makes the tests run much faster.
char* optimizations = getenv("EXPR_TEST_ENABLE_OPTIMIZATIONS");
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/literal.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/literal.cc b/be/src/exprs/literal.cc
index 3b7caf8..48fe7d2 100644
--- a/be/src/exprs/literal.cc
+++ b/be/src/exprs/literal.cc
@@ -176,8 +176,6 @@ Literal::Literal(ColumnType type, double v)
: ScalarExpr(type, true) {
if (type.type == TYPE_DOUBLE) {
value_.double_val = v;
- } else if (type.type == TYPE_TIMESTAMP) {
- value_.timestamp_val = TimestampValue::FromSubsecondUnixTime(v);
} else if (type.type == TYPE_DECIMAL) {
bool overflow = false;
switch (type.GetByteSize()) {
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index f2d741e..e328b9e 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -79,7 +79,8 @@ StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
template <class TIME>
StringVal TimestampFunctions::FromUnix(FunctionContext* context, const TIME& intp) {
if (intp.is_null) return StringVal::null();
- const TimestampValue tv = TimestampValue::FromUnixTime(intp.val);
+ const TimestampValue tv = TimestampValue::FromUnixTime(intp.val,
+ context->impl()->state()->local_time_zone());
if (!tv.HasDateAndTime()) return StringVal::null();
return AnyValUtil::FromString(context, tv.ToString());
}
@@ -93,7 +94,8 @@ StringVal TimestampFunctions::FromUnix(FunctionContext* context, const TIME& int
}
if (intp.is_null) return StringVal::null();
- const TimestampValue& t = TimestampValue::FromUnixTime(intp.val);
+ const TimestampValue& t = TimestampValue::FromUnixTime(intp.val,
+ context->impl()->state()->local_time_zone());
return StringValFromTimestamp(context, t, fmt);
}
@@ -103,19 +105,22 @@ BigIntVal TimestampFunctions::Unix(FunctionContext* context, const StringVal& st
if (tv_val.is_null) return BigIntVal::null();
const TimestampValue& tv = TimestampValue::FromTimestampVal(tv_val);
time_t result;
- return (tv.ToUnixTime(&result)) ? BigIntVal(result) : BigIntVal::null();
+ return (tv.ToUnixTime(context->impl()->state()->local_time_zone(), &result)) ?
+ BigIntVal(result) : BigIntVal::null();
}
BigIntVal TimestampFunctions::Unix(FunctionContext* context, const TimestampVal& ts_val) {
if (ts_val.is_null) return BigIntVal::null();
const TimestampValue& tv = TimestampValue::FromTimestampVal(ts_val);
time_t result;
- return (tv.ToUnixTime(&result)) ? BigIntVal(result) : BigIntVal::null();
+ return (tv.ToUnixTime(context->impl()->state()->local_time_zone(), &result)) ?
+ BigIntVal(result) : BigIntVal::null();
}
BigIntVal TimestampFunctions::Unix(FunctionContext* context) {
time_t result;
- if (context->impl()->state()->now()->ToUnixTime(&result)) {
+ if (context->impl()->state()->now()->ToUnixTime(
+ context->impl()->state()->local_time_zone(), &result)) {
return BigIntVal(result);
} else {
return BigIntVal::null();
@@ -142,7 +147,8 @@ TimestampVal TimestampFunctions::UnixMicrosToUtcTimestamp(FunctionContext* conte
TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
const BigIntVal& bigint_val) {
if (bigint_val.is_null) return TimestampVal::null();
- const TimestampValue& tv = TimestampValue::FromUnixTime(bigint_val.val);
+ const TimestampValue& tv = TimestampValue::FromUnixTime(bigint_val.val,
+ context->impl()->state()->local_time_zone());
TimestampVal tv_val;
tv.ToTimestampVal(&tv_val);
return tv_val;
@@ -185,7 +191,8 @@ BigIntVal TimestampFunctions::UnixFromString(FunctionContext* context,
const TimestampValue& tv = TimestampValue::Parse(
reinterpret_cast<const char *>(sv.ptr), sv.len);
time_t result;
- return (tv.ToUnixTime(&result)) ? BigIntVal(result) : BigIntVal::null();
+ return (tv.ToUnixTime(context->impl()->state()->local_time_zone(), &result)) ?
+ BigIntVal(result) : BigIntVal::null();
}
void TimestampFunctions::ReportBadFormat(FunctionContext* context,
@@ -493,6 +500,14 @@ StringVal TimestampFunctions::LongMonthName(FunctionContext* context,
return StringVal(reinterpret_cast<uint8_t*>(const_cast<char*>(mn.data())), mn.size());
}
+namespace {
+inline cctz::time_point<cctz::sys_seconds> UnixTimeToTimePoint(time_t t) {
+ return std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0)) + cctz::sys_seconds(t);
+}
+
+}
+
StringVal TimestampFunctions::TimeOfDay(FunctionContext* context) {
const TimestampVal curr = Now(context);
if (curr.is_null) return StringVal::null();
@@ -503,16 +518,23 @@ StringVal TimestampFunctions::TimeOfDay(FunctionContext* context) {
IntVal min = Minute(context, curr);
IntVal sec = Second(context, curr);
IntVal year = Year(context, curr);
- time_t rawtime;
- time(&rawtime);
- struct tm tzone;
- localtime_r(&rawtime, &tzone);
+
+ // Calculate 'start' time point at which query execution started.
+ cctz::time_point<cctz::sys_seconds> start = UnixTimeToTimePoint(
+ context->impl()->state()->query_ctx().start_unix_millis / MILLIS_PER_SEC);
+ // Find 'tz_name' time-zone abbreviation that corresponds to 'local_time_zone' at
+ // 'start' time point.
+ cctz::time_zone::absolute_lookup start_lookup =
+ context->impl()->state()->local_time_zone().lookup(start);
+ const string& tz_name = (start_lookup.abbr != nullptr) ? start_lookup.abbr :
+ context->impl()->state()->local_time_zone().name();
+
stringstream result;
result << day << " " << month << " " << setw(2) << setfill('0')
<< dayofmonth.val << " " << setw(2) << setfill('0') << hour.val << ":"
<< setw(2) << setfill('0') << min.val << ":"
<< setw(2) << setfill('0') << sec.val << " " << year.val << " "
- << tzone.tm_zone;
+ << tz_name;
return AnyValUtil::FromString(context, result.str());
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/timestamp-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
index 76edd9a..efc83cc 100644
--- a/be/src/exprs/timestamp-functions.cc
+++ b/be/src/exprs/timestamp-functions.cc
@@ -17,29 +17,19 @@
#include "exprs/timestamp-functions.h"
-#include <boost/algorithm/string.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <ctime>
-
+#include "cctz/time_zone.h"
#include "exprs/anyval-util.h"
#include "exprs/timezone_db.h"
#include "gutil/strings/substitute.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
+#include "runtime/timestamp-value.inline.h"
#include "udf/udf-internal.h"
#include "udf/udf.h"
#include "common/names.h"
-using boost::algorithm::iequals;
-using boost::local_time::local_date_time;
-using boost::local_time::time_zone_ptr;
-using boost::posix_time::ptime;
-using boost::posix_time::to_iso_extended_string;
-
-typedef boost::gregorian::date Date;
-
namespace impala {
const string TimestampFunctions::DAY_ARRAY[7] = {"Sun", "Mon", "Tue", "Wed", "Thu",
@@ -52,65 +42,39 @@ const string TimestampFunctions::MONTHNAME_ARRAY[12] = {"January", "February", "
"April", "May", "June", "July", "August", "September", "October", "November",
"December"};
-namespace {
-/// Uses Boost's internal checking to throw an exception if 'date' is out of the
-/// supported range of boost::gregorian.
-void ThrowIfDateOutOfRange(const boost::gregorian::date& date) {
- // Boost checks the ranges when instantiating the year/month/day representations.
- boost::gregorian::greg_year year = date.year();
- boost::gregorian::greg_month month = date.month();
- boost::gregorian::greg_day day = date.day();
- // Ensure Boost's validation is effective.
- DCHECK_GE(year, boost::gregorian::greg_year::min());
- DCHECK_LE(year, boost::gregorian::greg_year::max());
- DCHECK_GE(month, boost::gregorian::greg_month::min());
- DCHECK_LE(month, boost::gregorian::greg_month::max());
- DCHECK_GE(day, boost::gregorian::greg_day::min());
- DCHECK_LE(day, boost::gregorian::greg_day::max());
-}
-}
-
-// This function uses inline asm functions, which we believe to be from the boost library.
-// Inline asm is not currently supported by JIT, so this function should always be run in
-// the interpreted mode. This is handled in LlvmCodeGen::LoadFunction().
TimestampVal TimestampFunctions::FromUtc(FunctionContext* context,
const TimestampVal& ts_val, const StringVal& tz_string_val) {
if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
- const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
- if (!ts_value.HasDateAndTime()) return TimestampVal::null();
+ const TimestampValue ts_value = TimestampValue::FromTimestampVal(ts_val);
+ if (UNLIKELY(!ts_value.HasDateAndTime())) return TimestampVal::null();
const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
- time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
- string(tz_string_value.ptr, tz_string_value.len), ts_value, true);
- if (timezone == NULL) {
- // This should return null. Hive just ignores it.
+ const Timezone* timezone = TimezoneDatabase::FindTimezone(
+ string(tz_string_value.ptr, tz_string_value.len));
+ if (UNLIKELY(timezone == nullptr)) {
+ // Although this is an error, Hive ignores it. We will issue a warning but otherwise
+ // ignore the error too.
stringstream ss;
ss << "Unknown timezone '" << tz_string_value << "'" << endl;
context->AddWarning(ss.str().c_str());
return ts_val;
}
- try {
- ptime temp;
- ts_value.ToPtime(&temp);
- local_date_time lt(temp, timezone);
- ptime local_time = lt.local_time();
- ThrowIfDateOutOfRange(local_time.date());
- TimestampVal return_val;
- TimestampValue(local_time).ToTimestampVal(&return_val);
- return return_val;
- } catch (boost::exception&) {
- const string& msg = Substitute(
+ TimestampValue ts_value_ret = ts_value;
+ ts_value_ret.UtcToLocal(*timezone);
+ if (UNLIKELY(!ts_value_ret.HasDateAndTime())) {
+ const string msg = Substitute(
"Timestamp '$0' did not convert to a valid local time in timezone '$1'",
ts_value.ToString(), tz_string_value.DebugString());
context->AddWarning(msg.c_str());
return TimestampVal::null();
}
+
+ TimestampVal ts_val_ret;
+ ts_value_ret.ToTimestampVal(&ts_val_ret);
+ return ts_val_ret;
}
-// This function uses inline asm functions, which we believe to be from the boost library.
-// Inline asm is not currently supported by JIT, so this function should always be run in
-// the interpreted mode. This is handled in LlvmCodeGen::LoadFunction().
TimestampVal TimestampFunctions::ToUtc(FunctionContext* context,
const TimestampVal& ts_val, const StringVal& tz_string_val) {
if (ts_val.is_null || tz_string_val.is_null) return TimestampVal::null();
@@ -118,42 +82,30 @@ TimestampVal TimestampFunctions::ToUtc(FunctionContext* context,
if (!ts_value.HasDateAndTime()) return TimestampVal::null();
const StringValue& tz_string_value = StringValue::FromStringVal(tz_string_val);
- time_zone_ptr timezone = TimezoneDatabase::FindTimezone(
- string(tz_string_value.ptr, tz_string_value.len), ts_value, false);
- // This should raise some sort of error or at least null. Hive Just ignores it.
- if (UNLIKELY(timezone == NULL)) {
+ const Timezone* timezone = TimezoneDatabase::FindTimezone(
+ string(tz_string_value.ptr, tz_string_value.len));
+ if (UNLIKELY(timezone == nullptr)) {
+ // Although this is an error, Hive ignores it. We will issue a warning but otherwise
+ // ignore the error too.
stringstream ss;
ss << "Unknown timezone '" << tz_string_value << "'" << endl;
context->AddWarning(ss.str().c_str());
return ts_val;
}
- if (UNLIKELY(timezone == TimezoneDatabase::TIMEZONE_MSK_PRE_2014 &&
- ts_value.date().day_number() == 2456957 &&
- ts_value.time().hours() >= 1)) {
- // On October 27, 2014 at 1:00 am MSC, Moscow time transitions from UTC+4 with no DST
- // to UTC+3 with no DST. Because of this, 1am to 1:59:59.999...am MSC happens twice.
- // We want to be consistent with the existing rule of "in case of ambiguity return
- // NULL".
- DCHECK_LT(ts_value.time().hours(), 2);
- return TimestampVal::null();
- }
- try {
- local_date_time lt(ts_value.date(), ts_value.time(), timezone,
- local_date_time::NOT_DATE_TIME_ON_ERROR);
- ptime utc_time = lt.utc_time();
- // The utc_time() conversion does not check ranges - need to explicitly check.
- ThrowIfDateOutOfRange(utc_time.date());
- TimestampVal return_val;
- TimestampValue(utc_time).ToTimestampVal(&return_val);
- return return_val;
- } catch (boost::exception&) {
+ TimestampValue ts_value_ret = ts_value;
+ ts_value_ret.LocalToUtc(*timezone);
+ if (UNLIKELY(!ts_value_ret.HasDateAndTime())) {
const string& msg =
Substitute("Timestamp '$0' in timezone '$1' could not be converted to UTC",
ts_value.ToString(), tz_string_value.DebugString());
context->AddWarning(msg.c_str());
return TimestampVal::null();
}
+
+ TimestampVal ts_val_ret;
+ ts_value_ret.ToTimestampVal(&ts_val_ret);
+ return ts_val_ret;
}
void TimestampFunctions::UnixAndFromUnixPrepare(
@@ -194,52 +146,4 @@ void TimestampFunctions::UnixAndFromUnixClose(FunctionContext* context,
}
}
-time_zone_ptr TimezoneDatabase::FindTimezone(
- const string& tz, const TimestampValue& tv, bool tv_in_utc) {
- // The backing database does not handle timezone rule changes.
- if (iequals("Europe/Moscow", tz) || iequals("Moscow", tz) || iequals("MSK", tz)) {
- if (tv.date().year() < 2011 || (tv.date().year() == 2011 && tv.date().month() < 4)) {
- // Between January 19, 1992 and March 27, 2011 Moscow time was UTC+3 with DST. On
- // March 27, 2011 Moscow time transitioned to UTC+4 with no DST. NOTE: We currently
- // do not handle Moscow time conversions for dates before January 19, 1992
- // correctly (Impala incorrectly thinks the Moscow timezone is UTC+3 with DST
- // instead of UTC+2 with DST for those dates).
- return TIMEZONE_MSK_PRE_2011_DST;
- }
- // On October 26, 2014 at 22:00:00 UTC, Moscow time transitioned to UTC+3 with no
- // DST. We have to make a precise time check here, unlike in the case above, because
- // we can't rely on the timezone database to handle the moment of transition because
- // the rule change does not coincide with a DST change.
- const int MSK_TRANSITION_DAY = 2456956;
- const int MSK_TRANSITION_HOUR_UTC = 22;
- const int MSK_UTC_OFFSET = 4;
- if (tv.date().day_number() < MSK_TRANSITION_DAY) return TIMEZONE_MSK_PRE_2014;
- if (tv_in_utc) {
- if (tv.date().day_number() == MSK_TRANSITION_DAY &&
- tv.time().hours() < MSK_TRANSITION_HOUR_UTC) {
- return TIMEZONE_MSK_PRE_2014;
- }
- } else if (tv.date().day_number() < MSK_TRANSITION_DAY + 1 || (
- tv.date().day_number() == MSK_TRANSITION_DAY + 1 &&
- tv.time().hours() < (MSK_TRANSITION_HOUR_UTC + MSK_UTC_OFFSET) % 24)) {
- return TIMEZONE_MSK_PRE_2014;
- }
- }
-
- // See if they specified a zone id
- time_zone_ptr tzp = tz_database_.time_zone_from_region(tz);
- if (tzp != NULL) return tzp;
-
- for (vector<string>::const_iterator iter = tz_region_list_.begin();
- iter != tz_region_list_.end(); ++iter) {
- time_zone_ptr tzp = tz_database_.time_zone_from_region(*iter);
- DCHECK(tzp != NULL);
- if (tzp->dst_zone_abbrev() == tz) return tzp;
- if (tzp->std_zone_abbrev() == tz) return tzp;
- if (tzp->dst_zone_name() == tz) return tzp;
- if (tzp->std_zone_name() == tz) return tzp;
- }
- return time_zone_ptr();
-}
-
}