You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/06/22 18:11:15 UTC
[3/8] impala git commit: IMPALA-3307: Add support for IANA time-zone
db
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/exprs/timezone_db.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h
index 3a1178a..1a39344 100644
--- a/be/src/exprs/timezone_db.h
+++ b/be/src/exprs/timezone_db.h
@@ -15,43 +15,135 @@
// specific language governing permissions and limitations
// under the License.
-
#ifndef IMPALA_EXPRS_TIMEZONE_DB_H
#define IMPALA_EXPRS_TIMEZONE_DB_H
-#include <boost/date_time/compiler_config.hpp>
-#include <boost/date_time/local_time/local_time_types.hpp>
-#include <boost/date_time/local_time/tz_database.hpp>
+#include <unordered_map>
+#include "cctz/time_zone.h"
+#include "common/global-types.h"
#include "common/status.h"
-#include "runtime/timestamp-value.h"
+#include "util/zip-util.h"
namespace impala {
-/// Functions to load and access the timestamp database.
+/// 'TimezoneDatabase' class contains functions to load and access the IANA time-zone
+/// database. The IANA time-zone database (often called tz db) contains binary data files
+/// that represent the history of local time for many representative locations around the
+/// globe. Further information on tz db, including implementation details, is described in
+/// the code repository: https://data.iana.org/time-zones/tz-link.html
+///
+/// The system's default tz db is located in /usr/share/zoneinfo directory. The zoneinfo
+/// directory's tree structure follows the IANA time-zone naming scheme, e.g. the
+/// /usr/share/zoneinfo/America/Los_Angeles file describes the "America/Los_Angeles"
+/// time-zone.
+///
+/// This class uses the CCTZ library's 'time_zone' class internally to load, store and
+/// utilize time-zone data. Further information on CCTZ is described in the code
+/// repository: https://github.com/google/cctz
+///
+/// Initialize() should be called on process startup to load every time-zone file into
+/// memory for fast lookups. It loads files from /usr/share/zoneinfo by default.
+/// Alternatively, FLAGS_hdfs_zone_info_zip can be used to specify a shared zip file that
+/// contains the compiled time-zone db to use.
+///
+/// Initialize() also defines a hard-coded set of non-standard time-zone aliases to
+/// maintain a level of backward compatibility with the previous boost-based
+/// implementation. Alternatively, FLAGS_hdfs_zone_alias_conf can be used to specify a
+/// shared configuration file to load the non-standard aliases from.
+///
+/// Once Initialize() returned without error, FindTimezone() can be safely called from
+/// multiple threads to look up time-zones by name.
+
class TimezoneDatabase {
public:
- /// Set up the static timezone database.
+ /// Set up the static time-zone database.
static Status Initialize() WARN_UNUSED_RESULT;
- /// Converts the name of a timezone to a boost timezone object. In some cases, the
- /// timestamp is required to determine the timezone because occasionally timezone
- /// rules change. The flag "tv_in_utc" specifies whether the given timestamp value is
- /// in the UTC timezone. If the flag is false, then the timestamp value is assumed to
- /// be in the 'tz' timezone.
- static boost::local_time::time_zone_ptr FindTimezone(const std::string& tz,
- const TimestampValue& tv, bool tv_in_utc);
+ /// Return path to time-zone database.
+ static const string& GetPath() { return tz_db_path_; }
- /// Moscow timezone UTC+3 with DST, for use before March 27, 2011.
- static const boost::local_time::time_zone_ptr TIMEZONE_MSK_PRE_2011_DST;
+ /// Returns name of the local time-zone or empty string if cannot find it.
+ static std::string LocalZoneName();
- /// Moscow timezone UTC+4 with no DST, for use before October 26, 2014.
- static const boost::local_time::time_zone_ptr TIMEZONE_MSK_PRE_2014;
+ /// Looks up 'Timezone' object by name. Returns pointer to the 'Timezone' object if the
+ /// lookup was successful and nullptr otherwise.
+ static const Timezone* FindTimezone(const std::string& tz_name) {
+ auto it = tz_name_map_.find(tz_name);
+ return (it == tz_name_map_.end()) ? nullptr : it->second.get();
+ }
+
+ static const Timezone& GetUtcTimezone() { return UTC_TIMEZONE_; }
private:
- static const char* TIMEZONE_DATABASE_STR;
- static boost::local_time::tz_database tz_database_;
- static std::vector<std::string> tz_region_list_;
+ // For BE tests
+ friend class TimezoneDbNamesTest;
+ friend class TimezoneDbLoadAliasTest;
+ friend class TimezoneDbLoadZoneInfoTest;
+
+ static const std::string ZONE_INFO_DIR;
+ static const std::string TIMEZONE_ALIASES;
+ static const Timezone UTC_TIMEZONE_;
+
+ /// Type to map time-zone names to Timezone objects.
+ typedef std::unordered_map<std::string, std::shared_ptr<Timezone>> TimezoneMap;
+
+ static TimezoneMap tz_name_map_;
+ static std::string tz_db_path_;
+
+ /// Returns 'true' if 'tz_segment' is a valid time-zone name segment. Time-zone name
+ /// segments can have letters, digits and '_', '-', '+' characters only. Name segments
+ /// must begin with an uppercase letter.
+ /// Some examples of valid time-zone name segments are: Los_Angeles, GMT+1,
+ /// East-Indiana.
+ static bool IsTimezoneNameSegmentValid(
+ const std::string& tz_segment) WARN_UNUSED_RESULT;
+
+ /// Returns 'true' if 'tz_name' is a valid time-zone name. Time-zone names must be valid
+ /// time-zone name segments delimited by '/', e.g.: America/Argentina/San_Juan.
+ static bool IsTimezoneNameValid(const std::string& tz_name) WARN_UNUSED_RESULT;
+
+ /// Parses the UTC offset in 'tz_offset' and returns 'true' if it is valid. If a valid
+ /// offset was found, 'offset_sec' is set to the parsed value.
+ static bool IsTimezoneOffsetValid(const std::string& tz_offset,
+ int64_t* offset_sec) WARN_UNUSED_RESULT;
+
+ /// Load 'Timezone' objects into 'tz_name_map_' from the shared 'hdfs_zone_info_zip' zip
+ /// archive.
+ static Status LoadZoneInfoFromHdfs(const std::string& hdfs_zone_info_zip,
+ const std::string& local_dir) WARN_UNUSED_RESULT;
+
+ /// Load 'Timezone' objects into 'tz_name_map_' from 'zone_info_dir' path.
+ static Status LoadZoneInfo(const std::string& zone_info_dir) WARN_UNUSED_RESULT;
+
+ /// Recursive function to load 'Timezone' objects into 'tz_path_map' from 'path'.
+ /// 'zone_info_dir' is the root directory of the time-zone db.
+ static Status LoadZoneInfoHelper(const std::string& path,
+ const std::string& zone_info_dir, TimezoneMap& tz_path_map) WARN_UNUSED_RESULT;
+
+ /// Load 'Timezone' object from file 'path'. 'zone_info_dir' is the root directory of
+ /// the time-zone db.
+ /// - If 'path' is not a symbolic link, load 'Timezone' object from 'path' and add it to
+ /// 'tz_path_map' as a value mapped to 'path'.
+ /// - If 'path' is a symbolic link to another time-zone file, load 'Timezone' object
+ /// from the linked path and add it to 'tz_path_map' as a value mapped both to 'path'
+ /// and the linked path.
+ static void LoadTimezone(const std::string& path, const std::string& zone_info_dir,
+ TimezoneMap& tz_path_map);
+
+ /// Load 'Timezone' object from file 'path'. If successful, return 'shared_ptr' to the
+ /// 'Timezone' object and nullptr otherwise.
+ static std::shared_ptr<Timezone> LoadTimezoneHelper(
+ const std::string& path) WARN_UNUSED_RESULT;
+
+ /// Load custom time-zone aliases from 'hdfs_zone_alias_conf' shared file and add them
+ /// to 'tz_name_map_'.
+ static Status LoadZoneAliasesFromHdfs(
+ const string& hdfs_zone_alias_conf) WARN_UNUSED_RESULT;
+
+ /// Load custom time-zone aliases from 'is' and add them to 'tz_name_map_'.
+ static Status LoadZoneAliases(
+ std::istream &is, const char* path = nullptr) WARN_UNUSED_RESULT;
};
} // namespace impala
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/raw-value-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/raw-value-test.cc b/be/src/runtime/raw-value-test.cc
index bced866..5f00eb6 100644
--- a/be/src/runtime/raw-value-test.cc
+++ b/be/src/runtime/raw-value-test.cc
@@ -16,6 +16,8 @@
// under the License.
#include <sstream>
+
+#include "exprs/timezone_db.h"
#include "runtime/raw-value.inline.h"
#include "testutil/gtest-util.h"
@@ -173,7 +175,8 @@ TEST_F(RawValueTest, TemplatizedHash) {
RawValue::GetHashValue(&string_value,ColumnType::CreateVarcharType(
ColumnType::MAX_VARCHAR_LENGTH), seed));
- TimestampValue timestamp_value = TimestampValue::FromUnixTime(253433923200);
+ Timezone utc_tz = cctz::utc_time_zone();
+ TimestampValue timestamp_value = TimestampValue::FromUnixTime(253433923200, utc_tz);
EXPECT_EQ(RawValue::GetHashValue<impala::TimestampValue>(
×tamp_value, TYPE_TIMESTAMP, seed),RawValue::GetHashValue(
×tamp_value, TYPE_TIMESTAMP, seed));
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index 041c19b..6928ebc 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -31,6 +31,7 @@
#include "common/status.h"
#include "exprs/scalar-expr.h"
#include "exprs/scalar-fn-call.h"
+#include "exprs/timezone_db.h"
#include "runtime/bufferpool/buffer-pool.h"
#include "runtime/bufferpool/reservation-tracker.h"
#include "runtime/data-stream-mgr-base.h"
@@ -51,6 +52,7 @@
#include "util/jni-util.h"
#include "util/mem-info.h"
#include "util/pretty-printer.h"
+#include "util/test-info.h"
#include "common/names.h"
@@ -70,6 +72,7 @@ RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx& frag
now_(new TimestampValue(TimestampValue::Parse(query_state->query_ctx().now_string))),
utc_timestamp_(new TimestampValue(TimestampValue::Parse(
query_state->query_ctx().utc_timestamp_string))),
+ local_time_zone_(&TimezoneDatabase::GetUtcTimezone()),
exec_env_(exec_env),
profile_(RuntimeProfile::Create(
obj_pool(), "Fragment " + PrintId(instance_ctx.fragment_instance_id))),
@@ -87,6 +90,7 @@ RuntimeState::RuntimeState(
local_query_state_(query_state_),
now_(new TimestampValue(TimestampValue::Parse(qctx.now_string))),
utc_timestamp_(new TimestampValue(TimestampValue::Parse(qctx.utc_timestamp_string))),
+ local_time_zone_(&TimezoneDatabase::GetUtcTimezone()),
exec_env_(exec_env),
profile_(RuntimeProfile::Create(obj_pool(), "<unnamed>")) {
// We may use execution resources while evaluating exprs, etc. Decremented in
@@ -127,6 +131,23 @@ void RuntimeState::Init() {
query_state_->buffer_reservation(), instance_mem_tracker_.get(),
numeric_limits<int64_t>::max());
}
+
+ // Find local timezone.
+ // (For FE tests leave 'local_time_zone_' as default. FE tests don't load the timezone
+ // db since they don't need any timezone information.)
+ if (!TestInfo::is_fe_test()) {
+ const Timezone* tz = TimezoneDatabase::FindTimezone(query_ctx().local_time_zone);
+ if (tz != nullptr) {
+ local_time_zone_ = tz;
+ } else {
+ const string msg = Substitute(
+ "Failed to find local timezone '$0'.Falling back to UTC.",
+ query_ctx().local_time_zone);
+ LOG(WARNING) << msg;
+ LogError(ErrorMsg(TErrorCode::GENERAL, msg));
+ local_time_zone_ = &TimezoneDatabase::GetUtcTimezone();
+ }
+ }
}
Status RuntimeState::InitFilterBank(long runtime_filters_reservation_bytes) {
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 8b91c67..6d38fc6 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -99,6 +99,7 @@ class RuntimeState {
const TimestampValue* now() const { return now_.get(); }
const TimestampValue* utc_timestamp() const { return utc_timestamp_.get(); }
void set_now(const TimestampValue* now);
+ const Timezone& local_time_zone() const { return *local_time_zone_; }
const TUniqueId& query_id() const { return query_ctx().query_id; }
const TUniqueId& fragment_instance_id() const {
return instance_ctx_ != nullptr
@@ -322,6 +323,10 @@ class RuntimeState {
boost::scoped_ptr<TimestampValue> now_;
boost::scoped_ptr<TimestampValue> utc_timestamp_;
+ /// Query-global timezone used as local timezone when executing the query.
+ /// Owned by a static storage member of TimezoneDatabase class. It cannot be nullptr.
+ const Timezone* local_time_zone_;
+
/// TODO: get rid of this and use ExecEnv::GetInstance() instead
ExecEnv* exec_env_;
boost::scoped_ptr<LlvmCodeGen> codegen_;
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/timestamp-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-test.cc b/be/src/runtime/timestamp-test.cc
index 66bd896..8afa957 100644
--- a/be/src/runtime/timestamp-test.cc
+++ b/be/src/runtime/timestamp-test.cc
@@ -22,6 +22,7 @@
#include <boost/date_time/posix_time/posix_time.hpp>
#include "common/status.h"
+#include "exprs/timezone_db.h"
#include "runtime/raw-value.inline.h"
#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
@@ -234,6 +235,8 @@ TEST(TimestampTest, Basic) {
// Set it to 03/01 to test 02/29 edge cases.
TimestampValue now(date(1980, 3, 1), time_duration(16, 14, 24));
+ Timezone utc_tz = cctz::utc_time_zone();
+
char s1[] = "2012-01-20 01:10:01";
char s2[] = "1990-10-20 10:10:10.123456789 ";
char s3[] = " 1990-10-20 10:10:10.123456789";
@@ -615,7 +618,7 @@ TEST(TimestampTest, Basic) {
TimestampFormatTC test_case = fmt_test_cases[i];
DateTimeFormatContext dt_ctx(test_case.fmt, strlen(test_case.fmt));
ASSERT_TRUE(TimestampParser::ParseFormatTokens(&dt_ctx)) << "TC: " << i;
- TimestampValue cust_tv = TimestampValue::FromUnixTime(test_case.ts);
+ TimestampValue cust_tv = TimestampValue::FromUnixTime(test_case.ts, utc_tz);
EXPECT_NE(cust_tv.date(), not_a_date) << "TC: " << i;
EXPECT_NE(cust_tv.time(), not_a_date_time) << "TC: " << i;
EXPECT_GE(dt_ctx.fmt_out_len, dt_ctx.fmt_len);
@@ -634,29 +637,30 @@ TEST(TimestampTest, Basic) {
EXPECT_TRUE(min_date.HasDate());
EXPECT_TRUE(min_date.HasTime());
time_t tm_min;
- EXPECT_TRUE(min_date.ToUnixTime(&tm_min));
+ EXPECT_TRUE(min_date.ToUnixTime(utc_tz, &tm_min));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, tm_min);
int64_t tm_min_micros;
EXPECT_TRUE(min_date.UtcToUnixTimeMicros(&tm_min_micros));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, tm_min_micros);
// Add 250ns and check the value is rounded down
- min_date = TimestampValue::FromUnixTimeNanos(MIN_DATE_AS_UNIX_TIME, 250);
- EXPECT_TRUE(min_date.ToUnixTime(&tm_min));
+ min_date = TimestampValue::FromUnixTimeNanos(MIN_DATE_AS_UNIX_TIME, 250, utc_tz);
+ EXPECT_TRUE(min_date.ToUnixTime(utc_tz, &tm_min));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, tm_min);
EXPECT_TRUE(min_date.UtcToUnixTimeMicros(&tm_min_micros));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, tm_min_micros);
// Add another 250ns and check the value is rounded up to the nearest microsecond.
- EXPECT_TRUE(min_date.ToUnixTime(&tm_min));
+ EXPECT_TRUE(min_date.ToUnixTime(utc_tz, &tm_min));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, tm_min);
min_date.set_time(min_date.time() + boost::posix_time::nanoseconds(250));
EXPECT_TRUE(min_date.UtcToUnixTimeMicros(&tm_min_micros));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC + 1, tm_min_micros);
EXPECT_EQ("1400-01-01 00:00:00",
- TimestampValue::FromUnixTime(MIN_DATE_AS_UNIX_TIME).ToString());
- TimestampValue too_early = TimestampValue::FromUnixTime(MIN_DATE_AS_UNIX_TIME - 1);
+ TimestampValue::FromUnixTime(MIN_DATE_AS_UNIX_TIME, utc_tz).ToString());
+ TimestampValue too_early = TimestampValue::FromUnixTime(MIN_DATE_AS_UNIX_TIME - 1,
+ utc_tz);
EXPECT_FALSE(too_early.HasDate());
EXPECT_FALSE(too_early.HasTime());
EXPECT_FALSE(too_early.UtcToUnixTimeMicros(&tm_min_micros));
@@ -665,11 +669,11 @@ TEST(TimestampTest, Basic) {
// as valid, because validation logic checked the nearest second rounded towards 0
// (IMPALA-5664).
EXPECT_FALSE(TimestampValue::FromSubsecondUnixTime(
- MIN_DATE_AS_UNIX_TIME - 0.1).HasDate());
+ MIN_DATE_AS_UNIX_TIME - 0.1, utc_tz).HasDate());
EXPECT_FALSE(TimestampValue::UtcFromUnixTimeMicros(
MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC - 2000).HasDate());
EXPECT_FALSE(TimestampValue::FromUnixTimeNanos(
- MIN_DATE_AS_UNIX_TIME, -NANOS_PER_MICRO * 100).HasDate());
+ MIN_DATE_AS_UNIX_TIME, -NANOS_PER_MICRO * 100, utc_tz).HasDate());
// Test the max supported date that can be represented in seconds.
const int64_t MAX_DATE_AS_UNIX_TIME = 253402300799;
@@ -678,7 +682,7 @@ TEST(TimestampTest, Basic) {
EXPECT_TRUE(max_date.HasDate());
EXPECT_TRUE(max_date.HasTime());
time_t tm_max;
- EXPECT_TRUE(max_date.ToUnixTime(&tm_max));
+ EXPECT_TRUE(max_date.ToUnixTime(utc_tz, &tm_max));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME, tm_max);
int64_t tm_max_micros;
EXPECT_TRUE(max_date.UtcToUnixTimeMicros(&tm_max_micros));
@@ -696,7 +700,7 @@ TEST(TimestampTest, Basic) {
// The max date that can be represented with the maximum number of nanoseconds. Unlike
// the cases above, converting to microseconds does not round up to the next
// microsecond because that time is not supported by Impala.
- max_date = TimestampValue::FromUnixTimeNanos(MAX_DATE_AS_UNIX_TIME, 999999999);
+ max_date = TimestampValue::FromUnixTimeNanos(MAX_DATE_AS_UNIX_TIME, 999999999, utc_tz);
EXPECT_TRUE(max_date.HasDate());
EXPECT_TRUE(max_date.HasTime());
// The result is the maximum date with the maximum number of microseconds supported by
@@ -705,16 +709,17 @@ TEST(TimestampTest, Basic) {
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC + 999999, tm_max_micros);
EXPECT_EQ("9999-12-31 23:59:59",
- TimestampValue::FromUnixTime(MAX_DATE_AS_UNIX_TIME).ToString());
- TimestampValue too_late = TimestampValue::FromUnixTime(MAX_DATE_AS_UNIX_TIME + 1);
+ TimestampValue::FromUnixTime(MAX_DATE_AS_UNIX_TIME, utc_tz).ToString());
+ TimestampValue too_late = TimestampValue::FromUnixTime(MAX_DATE_AS_UNIX_TIME + 1,
+ utc_tz);
EXPECT_FALSE(too_late.HasDate());
EXPECT_FALSE(too_late.HasTime());
// Checking sub-second FromUnixTime functions near 10000.01.01
EXPECT_TRUE(TimestampValue::FromSubsecondUnixTime(
- MAX_DATE_AS_UNIX_TIME + 0.99).HasDate());
+ MAX_DATE_AS_UNIX_TIME + 0.99, utc_tz).HasDate());
EXPECT_FALSE(TimestampValue::FromSubsecondUnixTime(
- MAX_DATE_AS_UNIX_TIME + 1).HasDate());
+ MAX_DATE_AS_UNIX_TIME + 1, utc_tz).HasDate());
EXPECT_TRUE(TimestampValue::UtcFromUnixTimeMicros(
MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC + MICROS_PER_SEC - 1).HasDate());
@@ -722,31 +727,31 @@ TEST(TimestampTest, Basic) {
MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC + MICROS_PER_SEC).HasDate());
EXPECT_TRUE(TimestampValue::FromUnixTimeNanos(
- MAX_DATE_AS_UNIX_TIME, NANOS_PER_SEC - 1).HasDate());
+ MAX_DATE_AS_UNIX_TIME, NANOS_PER_SEC - 1, utc_tz).HasDate());
EXPECT_FALSE(TimestampValue::FromUnixTimeNanos(
- MAX_DATE_AS_UNIX_TIME, NANOS_PER_SEC).HasDate());
+ MAX_DATE_AS_UNIX_TIME, NANOS_PER_SEC, utc_tz).HasDate());
// Regression tests for IMPALA-1676, Unix times overflow int32 during year 2038
EXPECT_EQ("2038-01-19 03:14:08",
- TimestampValue::FromUnixTime(2147483648).ToString());
+ TimestampValue::FromUnixTime(2147483648, utc_tz).ToString());
EXPECT_EQ("2038-01-19 03:14:09",
- TimestampValue::FromUnixTime(2147483649).ToString());
+ TimestampValue::FromUnixTime(2147483649, utc_tz).ToString());
// Tests for the cases where abs(nanoseconds) >= 1e9.
EXPECT_EQ("2018-01-10 16:00:00",
- TimestampValue::FromUnixTimeNanos(1515600000, 0).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, 0, utc_tz).ToString());
EXPECT_EQ("2018-01-10 16:00:00.999999999",
- TimestampValue::FromUnixTimeNanos(1515600000, 999999999).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, 999999999, utc_tz).ToString());
EXPECT_EQ("2018-01-10 15:59:59.000000001",
- TimestampValue::FromUnixTimeNanos(1515600000, -999999999).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, -999999999, utc_tz).ToString());
EXPECT_EQ("2018-01-10 16:00:01",
- TimestampValue::FromUnixTimeNanos(1515600000, 1000000000).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, 1000000000, utc_tz).ToString());
EXPECT_EQ("2018-01-10 15:59:59",
- TimestampValue::FromUnixTimeNanos(1515600000, -1000000000).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, -1000000000, utc_tz).ToString());
EXPECT_EQ("2018-01-10 16:30:00",
- TimestampValue::FromUnixTimeNanos(1515600000, 1800000000000).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, 1800000000000, utc_tz).ToString());
EXPECT_EQ("2018-01-10 15:30:00",
- TimestampValue::FromUnixTimeNanos(1515600000, -1800000000000).ToString());
+ TimestampValue::FromUnixTimeNanos(1515600000, -1800000000000, utc_tz).ToString());
// Test FromUnixTime around the boundary of the values that are converted via boost via
// gmtime (IMPALA-5357). Tests 1 second before and after the values supported by the
@@ -754,20 +759,20 @@ TEST(TimestampTest, Basic) {
const int64_t MIN_BOOST_CONVERT_UNIX_TIME = -9223372036;
const int64_t MAX_BOOST_CONVERT_UNIX_TIME = 9223372036;
EXPECT_EQ("1677-09-21 00:12:43",
- TimestampValue::FromUnixTime(MIN_BOOST_CONVERT_UNIX_TIME - 1).ToString());
+ TimestampValue::FromUnixTime(MIN_BOOST_CONVERT_UNIX_TIME - 1, utc_tz).ToString());
EXPECT_EQ("1677-09-21 00:12:44",
- TimestampValue::FromUnixTime(MIN_BOOST_CONVERT_UNIX_TIME).ToString());
+ TimestampValue::FromUnixTime(MIN_BOOST_CONVERT_UNIX_TIME, utc_tz).ToString());
EXPECT_EQ("2262-04-11 23:47:16",
- TimestampValue::FromUnixTime(MAX_BOOST_CONVERT_UNIX_TIME).ToString());
+ TimestampValue::FromUnixTime(MAX_BOOST_CONVERT_UNIX_TIME, utc_tz).ToString());
EXPECT_EQ("2262-04-11 23:47:17",
- TimestampValue::FromUnixTime(MAX_BOOST_CONVERT_UNIX_TIME + 1).ToString());
+ TimestampValue::FromUnixTime(MAX_BOOST_CONVERT_UNIX_TIME + 1, utc_tz).ToString());
// Test a leap second in 1998 represented by the UTC time 1998-12-31 23:59:60.
// Unix time cannot represent the leap second, which repeats 915148800.
EXPECT_EQ("1998-12-31 23:59:59",
- TimestampValue::FromUnixTime(915148799).ToString());
+ TimestampValue::FromUnixTime(915148799, utc_tz).ToString());
EXPECT_EQ("1999-01-01 00:00:00",
- TimestampValue::FromUnixTime(915148800).ToString());
+ TimestampValue::FromUnixTime(915148800, utc_tz).ToString());
// The leap second doesn't parse in Impala.
TimestampValue leap_tv = TimestampValue::Parse("1998-12-31 23:59:60.00");
EXPECT_FALSE(leap_tv.HasDateAndTime());
@@ -782,20 +787,21 @@ TEST(TimestampTest, Basic) {
EXPECT_TRUE(leap_tv1.HasDateAndTime());
EXPECT_TRUE(leap_tv1 == TimestampValue(leap_ptime2));
time_t leap_time_t;
- EXPECT_TRUE(leap_tv1.ToUnixTime(&leap_time_t));
+ EXPECT_TRUE(leap_tv1.ToUnixTime(utc_tz, &leap_time_t));
EXPECT_EQ(915148800, leap_time_t);
EXPECT_EQ("1999-01-01 00:00:00", leap_tv1.ToString());
- EXPECT_TRUE(leap_tv2.ToUnixTime(&leap_time_t));
+ EXPECT_TRUE(leap_tv2.ToUnixTime(utc_tz, &leap_time_t));
EXPECT_EQ(915148800, leap_time_t);
EXPECT_EQ("1999-01-01 00:00:00", leap_tv2.ToString());
// Test Unix time as a float
double result;
EXPECT_TRUE(
- TimestampValue::Parse("2013-10-21 06:43:12.07").ToSubsecondUnixTime(&result));
+ TimestampValue::Parse("2013-10-21 06:43:12.07").ToSubsecondUnixTime(
+ utc_tz, &result));
EXPECT_EQ(1382337792.07, result);
EXPECT_EQ("1970-01-01 00:00:00.008000000",
- TimestampValue::FromSubsecondUnixTime(0.008).ToString());
+ TimestampValue::FromSubsecondUnixTime(0.008, utc_tz).ToString());
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/timestamp-value.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.cc b/be/src/runtime/timestamp-value.cc
index 20d6cee..50d1ee4 100644
--- a/be/src/runtime/timestamp-value.cc
+++ b/be/src/runtime/timestamp-value.cc
@@ -19,7 +19,11 @@
#include <boost/date_time/posix_time/posix_time.hpp>
+#include "exprs/timestamp-functions.h"
+#include "exprs/timezone_db.h"
#include "runtime/timestamp-parse-util.h"
+#include "runtime/timestamp-value.h"
+#include "runtime/timestamp-value.inline.h"
#include "common/names.h"
@@ -38,15 +42,6 @@ DEFINE_bool(use_local_tz_for_unix_timestamp_conversions, false,
"and from Unix times. When false, TIMESTAMPs are interpreted in the UTC time zone. "
"Set to true for Hive compatibility.");
-// Constants for use with Unix times. Leap-seconds do not apply.
-const int32_t SECONDS_IN_MINUTE = 60;
-const int32_t SECONDS_IN_HOUR = 60 * SECONDS_IN_MINUTE;
-const int32_t SECONDS_IN_DAY = 24 * SECONDS_IN_HOUR;
-
-// struct tm stores month/year data as an offset
-const unsigned short TM_YEAR_OFFSET = 1900;
-const unsigned short TM_MONTH_OFFSET = 1;
-
// Boost stores dates as an uint32_t. Since subtraction is needed, convert to signed.
const int64_t EPOCH_DAY_NUMBER =
static_cast<int64_t>(date(1970, boost::gregorian::Jan, 1).day_number());
@@ -78,74 +73,110 @@ int TimestampValue::Format(const DateTimeFormatContext& dt_ctx, int len, char* b
return TimestampParser::Format(dt_ctx, date_, time_, len, buff);
}
-void TimestampValue::UtcToLocal() {
+namespace {
+inline cctz::time_point<cctz::sys_seconds> UnixTimeToTimePoint(time_t t) {
+ static const cctz::time_point<cctz::sys_seconds> epoch =
+ std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0));
+ return epoch + cctz::sys_seconds(t);
+}
+
+inline time_t TimePointToUnixTime(const cctz::time_point<cctz::sys_seconds>& tp) {
+ static const cctz::time_point<cctz::sys_seconds> epoch =
+ std::chrono::time_point_cast<cctz::sys_seconds>(
+ std::chrono::system_clock::from_time_t(0));
+ return (tp - epoch).count();
+}
+
+// Returns 'true' iff 'cs' is out of valid range (years 1400..9999 are considered valid).
+inline bool IsDateOutOfRange(const cctz::civil_second& cs) {
+ // Smallest valid year.
+ const static int MIN_YEAR =
+ boost::gregorian::date(boost::date_time::min_date_time).year();
+ // Largest valid year.
+ const static int MAX_YEAR =
+ boost::gregorian::date(boost::date_time::max_date_time).year();
+ return cs.year() < MIN_YEAR || cs.year() > MAX_YEAR;
+}
+
+}
+
+void TimestampValue::UtcToLocal(const Timezone& local_tz) {
DCHECK(HasDateAndTime());
- // Previously, conversion was done using boost functions but it was found to be
- // too slow. Doing the conversion without function calls (which also avoids some
- // unnecessary validations) the conversion only take half as long. Original:
- // http://www.boost.org/doc/libs/1_55_0/boost/date_time/c_local_time_adjustor.hpp
- try {
- time_t utc =
- (static_cast<int64_t>(date_.day_number()) - EPOCH_DAY_NUMBER) * SECONDS_IN_DAY +
- time_.hours() * SECONDS_IN_HOUR +
- time_.minutes() * SECONDS_IN_MINUTE +
- time_.seconds();
- tm temp;
- if (UNLIKELY(localtime_r(&utc, &temp) == nullptr)) {
- *this = ptime(not_a_date_time);
- return;
+ time_t unix_time;
+ if (UNLIKELY(!UtcToUnixTime(&unix_time))) {
+ SetToInvalidDateTime();
+ } else {
+ cctz::time_point<cctz::sys_seconds> from_tp = UnixTimeToTimePoint(unix_time);
+ cctz::civil_second to_cs = cctz::convert(from_tp, local_tz);
+ // boost::gregorian::date() throws boost::gregorian::bad_year if year is not in the
+ // 1400..9999 range. Need to check validity before creating the date object.
+ if (UNLIKELY(IsDateOutOfRange(to_cs))) {
+ SetToInvalidDateTime();
+ } else {
+ date_ = boost::gregorian::date(to_cs.year(), to_cs.month(), to_cs.day());
+ // Time-zone conversion rules don't affect fractional seconds, leave them intact.
+ time_ = boost::posix_time::time_duration(to_cs.hour(), to_cs.minute(),
+ to_cs.second(), time_.fractional_seconds());
}
- // Unlikely but a time zone conversion may push the value over the min/max
- // boundary resulting in an exception.
- date_ = boost::gregorian::date(
- static_cast<unsigned short>(temp.tm_year + TM_YEAR_OFFSET),
- static_cast<unsigned short>(temp.tm_mon + TM_MONTH_OFFSET),
- static_cast<unsigned short>(temp.tm_mday));
- time_ = time_duration(temp.tm_hour, temp.tm_min, temp.tm_sec,
- time().fractional_seconds());
- } catch (std::exception& /* from Boost */) {
- *this = ptime(not_a_date_time);
}
}
-ostream& operator<<(ostream& os, const TimestampValue& timestamp_value) {
- return os << timestamp_value.ToString();
+void TimestampValue::LocalToUtc(const Timezone& local_tz) {
+ DCHECK(HasDateAndTime());
+ const cctz::civil_second from_cs(date_.year(), date_.month(), date_.day(),
+ time_.hours(), time_.minutes(), time_.seconds());
+
+ // 'from_cl' represents the 'time_point' that corresponds to 'from_cs' civil time within
+ // 'local_tz' time-zone.
+ const cctz::time_zone::civil_lookup from_cl = local_tz.lookup(from_cs);
+
+ // In case the resulting 'time_point' is ambiguous, we have to invalidate
+ // TimestampValue.
+ // 'civil_lookup' members and the details of handling ambiguity are described at:
+ // https://github.com/google/cctz/blob/a2dd3d0fbc811fe0a1d4d2dbb0341f1a3d28cb2a/
+ // include/cctz/time_zone.h#L106
+ if (UNLIKELY(from_cl.kind != cctz::time_zone::civil_lookup::UNIQUE)) {
+ SetToInvalidDateTime();
+ } else {
+ int64_t nanos = time_.fractional_seconds();
+ *this = UnixTimeToUtcPtime(TimePointToUnixTime(from_cl.pre));
+ // Time-zone conversion rules don't affect fractional seconds, leave them intact.
+ time_ += nanoseconds(nanos);
+ }
}
-void TimestampValue::Validate() {
- if (HasDate() && UNLIKELY(!IsValidDate(date_))) {
- time_ = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
- date_ = boost::gregorian::date(boost::gregorian::not_a_date_time);
- }
+ostream& operator<<(ostream& os, const TimestampValue& timestamp_value) {
+ return os << timestamp_value.ToString();
}
/// Return a ptime representation of the given Unix time (seconds since the Unix epoch).
-/// The time zone of the resulting ptime is local time. This is called by
-/// UnixTimeToPtime.
-ptime TimestampValue::UnixTimeToLocalPtime(time_t unix_time) {
- tm temp_tm;
- // TODO: avoid localtime*, which takes a global timezone db lock
- if (UNLIKELY(localtime_r(&unix_time, &temp_tm) == nullptr)) {
- return ptime(not_a_date_time);
- }
- try {
- return ptime_from_tm(temp_tm);
- } catch (std::exception&) {
+/// The time zone of the resulting ptime is 'local_tz'. This is called by UnixTimeToPtime.
+ptime TimestampValue::UnixTimeToLocalPtime(time_t unix_time,
+ const Timezone& local_tz) {
+ cctz::time_point<cctz::sys_seconds> from_tp = UnixTimeToTimePoint(unix_time);
+ cctz::civil_second to_cs = cctz::convert(from_tp, local_tz);
+ // boost::gregorian::date() throws boost::gregorian::bad_year if year is not in the
+ // 1400..9999 range. Need to check validity before creating the date object.
+ if (UNLIKELY(IsDateOutOfRange(to_cs))) {
return ptime(not_a_date_time);
+ } else {
+ return ptime(
+ boost::gregorian::date(to_cs.year(), to_cs.month(), to_cs.day()),
+ boost::posix_time::time_duration(to_cs.hour(), to_cs.minute(), to_cs.second()));
}
}
/// Return a ptime representation of the given Unix time (seconds since the Unix epoch).
/// The time zone of the resulting ptime is UTC.
-/// In order to avoid a serious performance degredation using libc (IMPALA-5357), this
-/// function uses boost to convert the time_t to a ptime. Unfortunately, because the boost
-/// conversion relies on time_duration to represent the time_t and internally
+/// In order to avoid a serious performance degredation using CCTZ, this function uses
+/// boost to convert the time_t to a ptime. Unfortunately, because the boost conversion
+/// relies on time_duration to represent the time_t and internally
/// time_duration stores nanosecond precision ticks, the 'fast path' conversion using
/// boost can only handle a limited range of dates (appx years 1677-2622, while Impala
-/// supports years 1600-9999). For dates outside this range, the conversion will instead
-/// use the libc function gmtime_r which supports those dates but takes the global lock
-/// for the timezone db (even though technically it is not needed for the conversion,
-/// again see IMPALA-5357). This is called by UnixTimeToPtime.
+/// supports years 1400-9999). For dates outside this range, the conversion will instead
+/// use the CCTZ function convert which supports those dates. This is called by
+/// UnixTimeToPtime.
ptime TimestampValue::UnixTimeToUtcPtime(time_t unix_time) {
// Minimum Unix time that can be converted with from_time_t: 1677-Sep-21 00:12:44
const int64_t MIN_BOOST_CONVERT_UNIX_TIME = -9223372036;
@@ -160,20 +191,12 @@ ptime TimestampValue::UnixTimeToUtcPtime(time_t unix_time) {
}
}
- tm temp_tm;
- if (UNLIKELY(gmtime_r(&unix_time, &temp_tm) == nullptr)) {
- return ptime(not_a_date_time);
- }
- try {
- return ptime_from_tm(temp_tm);
- } catch (std::exception&) {
- return ptime(not_a_date_time);
- }
+ return UnixTimeToLocalPtime(unix_time, TimezoneDatabase::GetUtcTimezone());
}
-ptime TimestampValue::UnixTimeToPtime(time_t unix_time) {
+ptime TimestampValue::UnixTimeToPtime(time_t unix_time, const Timezone& local_tz) {
if (FLAGS_use_local_tz_for_unix_timestamp_conversions) {
- return UnixTimeToLocalPtime(unix_time);
+ return UnixTimeToLocalPtime(unix_time, local_tz);
} else {
return UnixTimeToUtcPtime(unix_time);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/timestamp-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index 5a5e733..2509f21 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -22,10 +22,10 @@
#include <boost/date_time/compiler_config.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/date_time/local_time/local_time.hpp>
-#include <ctime>
#include <gflags/gflags.h>
#include <string>
+#include "common/global-types.h"
#include "gen-cpp/Data_types.h"
#include "udf/udf.h"
#include "util/hash-util.h"
@@ -92,36 +92,40 @@ class TimestampValue {
const DateTimeFormatContext& dt_ctx);
/// Unix time (seconds since 1970-01-01 UTC by definition) constructors.
- /// Return the corresponding timestamp in the local timezone if
+ /// Return the corresponding timestamp in the 'local_tz' time zone if
/// FLAGS_use_local_tz_for_unix_timestamp_conversions is true. Otherwise, return the
/// corresponding timestamp in UTC.
- static TimestampValue FromUnixTime(time_t unix_time) {
- return TimestampValue(UnixTimeToPtime(unix_time));
+ static TimestampValue FromUnixTime(time_t unix_time, const Timezone& local_tz) {
+ return TimestampValue(UnixTimeToPtime(unix_time, local_tz));
}
/// Same as FromUnixTime() above, but adds the specified number of nanoseconds to the
/// resulting TimestampValue. Handles negative nanoseconds and the case where
/// abs(nanos) >= 1e9.
- static TimestampValue FromUnixTimeNanos(time_t unix_time, int64_t nanos) {
- boost::posix_time::ptime temp = UnixTimeToPtime(unix_time);
+ static TimestampValue FromUnixTimeNanos(time_t unix_time, int64_t nanos,
+ const Timezone& local_tz) {
+ boost::posix_time::ptime temp = UnixTimeToPtime(unix_time, local_tz);
temp += boost::posix_time::nanoseconds(nanos);
return TimestampValue(temp);
}
- /// Return the corresponding timestamp in local time zone for the Unix time specified in
- /// microseconds.
- static TimestampValue FromUnixTimeMicros(int64_t unix_time_micros);
+ /// Return the corresponding timestamp in 'local_tz' time zone for the Unix time
+ /// specified in microseconds.
+ static TimestampValue FromUnixTimeMicros(int64_t unix_time_micros,
+ const Timezone& local_tz);
/// Return the corresponding timestamp in UTC for the Unix time specified in
/// microseconds.
static TimestampValue UtcFromUnixTimeMicros(int64_t unix_time_micros);
- /// Returns a TimestampValue where the integer part of the specified 'unix_time'
- /// specifies the number of seconds (see above), and the fractional part is converted
- /// to nanoseconds and added to the resulting TimestampValue.
- static TimestampValue FromSubsecondUnixTime(double unix_time) {
+ /// Returns a TimestampValue in 'local_tz' time zone where the integer part of the
+ /// specified 'unix_time' specifies the number of seconds (see above), and the
+ /// fractional part is converted to nanoseconds and added to the resulting
+ /// TimestampValue.
+ static TimestampValue FromSubsecondUnixTime(double unix_time,
+ const Timezone& local_tz) {
const time_t unix_time_whole = unix_time;
- boost::posix_time::ptime temp = UnixTimeToPtime(unix_time_whole);
+ boost::posix_time::ptime temp = UnixTimeToPtime(unix_time_whole, local_tz);
temp += boost::posix_time::nanoseconds((unix_time - unix_time_whole) / ONE_BILLIONTH);
return TimestampValue(temp);
}
@@ -133,6 +137,7 @@ class TimestampValue {
TimestampValue value;
memcpy(&value.date_, &udf_value.date, sizeof(value.date_));
memcpy(&value.time_, &udf_value.time_of_day, sizeof(value.time_));
+ value.Validate();
return value;
}
@@ -163,6 +168,7 @@ class TimestampValue {
static TimestampValue FromTColumnValue(const TColumnValue& tvalue) {
TimestampValue value;
memcpy(&value, tvalue.timestamp_val.c_str(), Size());
+ value.Validate();
return value;
}
@@ -201,29 +207,34 @@ class TimestampValue {
bool UtcToUnixTime(time_t* unix_time) const;
/// Interpret 'this' as a timestamp in UTC and convert to unix time in microseconds.
- /// Nanoseconds are rounded to the nearest microsecond supported by Impala. Returns
- /// false if the conversion failed ('unix_time_micros' will be undefined), otherwise
- /// true.
+ /// Nanoseconds are rounded to the nearest microsecond supported by Impala.
+ /// Returns false if the conversion failed ('unix_time_micros' will be undefined),
+ /// otherwise true.
bool UtcToUnixTimeMicros(int64_t* unix_time_micros) const;
- /// Converts to Unix time (seconds since the Unix epoch) representation. The time
- /// zone interpretation of the TimestampValue instance is determined by
- /// FLAGS_use_local_tz_for_unix_timestamp_conversions. If the flag is true, the
- /// instance is interpreted as a local value. If the flag is false, UTC is assumed.
- /// Returns false if the conversion failed (unix_time will be undefined), otherwise
- /// true.
- bool ToUnixTime(time_t* unix_time) const;
+ /// Converts to Unix time (seconds since the Unix epoch) representation. The time zone
+ /// interpretation of the TimestampValue instance is determined by
+ /// FLAGS_use_local_tz_for_unix_timestamp_conversions. If the flag is true, the instance
+ /// is interpreted as a value in the 'local_tz' time zone. If the flag is false, UTC is
+ /// assumed. Returns false if the conversion failed (unix_time will be undefined),
+ /// otherwise true.
+ bool ToUnixTime(const Timezone& local_tz, time_t* unix_time) const;
- /// Converts to Unix time with fractional seconds.
+ /// Converts to Unix time with fractional seconds. The time zone interpretation of the
+ /// TimestampValue instance is determined a above.
/// Returns false if the conversion failed (unix_time will be undefined), otherwise
/// true.
- bool ToSubsecondUnixTime(double* unix_time) const;
+ bool ToSubsecondUnixTime(const Timezone& local_tz, double* unix_time) const;
+
+ /// Converts from UTC to 'local_tz' time zone in-place. The caller must ensure the
+ /// TimestampValue this function is called upon has both a valid date and time.
+ void UtcToLocal(const Timezone& local_tz);
- /// Converts from UTC to local time in-place. The caller must ensure the TimestampValue
- /// this function is called upon has both a valid date and time.
- void UtcToLocal();
+ /// Converts from 'local_tz' to UTC time zone in-place. The caller must ensure the
+ /// TimestampValue this function is called upon has both a valid date and time.
+ void LocalToUtc(const Timezone& local_tz);
- void set_date(const boost::gregorian::date d) { date_ = d; }
+ void set_date(const boost::gregorian::date d) { date_ = d; Validate(); }
void set_time(const boost::posix_time::time_duration t) { time_ = t; }
const boost::gregorian::date& date() const { return date_; }
const boost::posix_time::time_duration& time() const { return time_; }
@@ -231,6 +242,7 @@ class TimestampValue {
TimestampValue& operator=(const boost::posix_time::ptime& ptime) {
date_ = ptime.date();
time_ = ptime.time_of_day();
+ Validate();
return *this;
}
@@ -284,18 +296,28 @@ class TimestampValue {
/// 4 -bytes - stores the date as a day
boost::gregorian::date date_;
+ /// Sets both date and time to invalid value.
+ inline void SetToInvalidDateTime() {
+ time_ = boost::posix_time::time_duration(boost::posix_time::not_a_date_time);
+ date_ = boost::gregorian::date(boost::gregorian::not_a_date_time);
+ }
+
/// Sets both date and time to invalid if date is outside the valid range.
- void Validate();
+ inline void Validate() {
+ if (HasDate() && UNLIKELY(!IsValidDate(date_))) SetToInvalidDateTime();
+ }
/// Return a ptime representation of the given Unix time (seconds since the Unix epoch).
/// The time zone of the resulting ptime is determined by
/// FLAGS_use_local_tz_for_unix_timestamp_conversions. If the flag is true, the value
- /// will be in the local time zone. If the flag is false, the value will be in UTC.
- static boost::posix_time::ptime UnixTimeToPtime(time_t unix_time);
-
- /// Same as the above, but the time zone of the resulting ptime is always in the local
- /// time zone.
- static boost::posix_time::ptime UnixTimeToLocalPtime(time_t unix_time);
+ /// will be in the 'local_tz' time zone. If the flag is false, the value will be in UTC.
+ static boost::posix_time::ptime UnixTimeToPtime(time_t unix_time,
+ const Timezone& local_tz);
+
+ /// Same as the above, but the time zone of the resulting ptime is always in the
+ /// 'local_tz' time zone.
+ static boost::posix_time::ptime UnixTimeToLocalPtime(time_t unix_time,
+ const Timezone& local_tz);
/// Same as the above, but the time zone of the resulting ptime is always in UTC.
static boost::posix_time::ptime UnixTimeToUtcPtime(time_t unix_time);
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/runtime/timestamp-value.inline.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.inline.h b/be/src/runtime/timestamp-value.inline.h
index cdabc20..e049147 100644
--- a/be/src/runtime/timestamp-value.inline.h
+++ b/be/src/runtime/timestamp-value.inline.h
@@ -23,7 +23,9 @@
#include <boost/date_time/compiler_config.hpp>
#include <boost/date_time/posix_time/conversion.hpp>
+#include <chrono>
+#include "exprs/timezone_db.h"
#include "gutil/walltime.h"
namespace impala {
@@ -36,10 +38,11 @@ inline TimestampValue TimestampValue::UtcFromUnixTimeMicros(int64_t unix_time_mi
return TimestampValue(temp);
}
-inline TimestampValue TimestampValue::FromUnixTimeMicros(int64_t unix_time_micros) {
+inline TimestampValue TimestampValue::FromUnixTimeMicros(int64_t unix_time_micros,
+ const Timezone& local_tz) {
int64_t ts_seconds = unix_time_micros / MICROS_PER_SEC;
int64_t micros_part = unix_time_micros - (ts_seconds * MICROS_PER_SEC);
- boost::posix_time::ptime temp = UnixTimeToLocalPtime(ts_seconds);
+ boost::posix_time::ptime temp = UnixTimeToLocalPtime(ts_seconds, local_tz);
temp += boost::posix_time::microseconds(micros_part);
return TimestampValue(temp);
}
@@ -50,10 +53,9 @@ inline TimestampValue TimestampValue::FromUnixTimeMicros(int64_t unix_time_micro
inline bool TimestampValue::UtcToUnixTime(time_t* unix_time) const {
DCHECK(unix_time != nullptr);
if (UNLIKELY(!HasDateAndTime())) return false;
- const boost::posix_time::ptime temp(date_, time_);
- tm temp_tm = boost::posix_time::to_tm(temp);
- // TODO: Conversion using libc is very expensive (IMPALA-5357); find an alternative.
- *unix_time = timegm(&temp_tm);
+
+ static const boost::gregorian::date epoch(1970,1,1);
+ *unix_time = (date_ - epoch).days() * 24 * 60 * 60 + time_.total_seconds();
return true;
}
@@ -64,12 +66,13 @@ inline bool TimestampValue::UtcToUnixTime(time_t* unix_time) const {
inline bool TimestampValue::UtcToUnixTimeMicros(int64_t* unix_time_micros) const {
const static int64_t MAX_UNIXTIME = 253402300799; // 9999-12-31 23:59:59
const static int64_t MAX_UNIXTIME_MICROS =
- MAX_UNIXTIME * MICROS_PER_SEC + (MICROS_PER_SEC - 1);
+ MAX_UNIXTIME * MICROS_PER_SEC + (MICROS_PER_SEC - 1);
DCHECK(unix_time_micros != nullptr);
time_t unixtime_seconds;
if (UNLIKELY(!UtcToUnixTime(&unixtime_seconds))) return false;
+ DCHECK(HasTime());
*unix_time_micros =
(static_cast<int64_t>(unixtime_seconds) * MICROS_PER_SEC) +
((time_.fractional_seconds() + (NANOS_PER_MICRO / 2)) / NANOS_PER_MICRO);
@@ -84,31 +87,36 @@ inline bool TimestampValue::UtcToUnixTimeMicros(int64_t* unix_time_micros) const
/// Converts to Unix time (seconds since the Unix epoch) representation. The time
/// zone interpretation of the TimestampValue instance is determined by
/// FLAGS_use_local_tz_for_unix_timestamp_conversions. If the flag is true, the
-/// instance is interpreted as a local value. If the flag is false, UTC is assumed.
+/// instance is interpreted as a value in the 'local_tz' time zone. If the flag is false,
+/// UTC is assumed.
/// Returns false if the conversion failed (unix_time will be undefined), otherwise
/// true.
-inline bool TimestampValue::ToUnixTime(time_t* unix_time) const {
+inline bool TimestampValue::ToUnixTime(const Timezone& local_tz,
+ time_t* unix_time) const {
DCHECK(unix_time != nullptr);
if (UNLIKELY(!HasDateAndTime())) return false;
- const boost::posix_time::ptime temp(date_, time_);
- tm temp_tm = boost::posix_time::to_tm(temp);
- // TODO: Conversion using libc is very expensive (IMPALA-5357); find an alternative.
- if (FLAGS_use_local_tz_for_unix_timestamp_conversions) {
- *unix_time = mktime(&temp_tm);
- } else {
- *unix_time = timegm(&temp_tm);
- }
+
+ if (!FLAGS_use_local_tz_for_unix_timestamp_conversions) return UtcToUnixTime(unix_time);
+
+ cctz::civil_second cs(date_.year(), date_.month(), date_.day(), time_.hours(),
+ time_.minutes(), time_.seconds());
+ cctz::time_point<cctz::sys_seconds> tp = cctz::convert(cs, local_tz);
+ cctz::sys_seconds seconds = tp.time_since_epoch();
+ *unix_time = seconds.count();
return true;
}
-/// Converts to Unix time with fractional seconds.
+/// Converts to Unix time with fractional seconds. The time zone interpretation of the
+/// TimestampValue instance is determined as above.
/// Returns false if the conversion failed (unix_time will be undefined), otherwise
/// true.
-inline bool TimestampValue::ToSubsecondUnixTime(double* unix_time) const {
+inline bool TimestampValue::ToSubsecondUnixTime(const Timezone& local_tz,
+ double* unix_time) const {
DCHECK(unix_time != nullptr);
time_t temp;
- if (UNLIKELY(!ToUnixTime(&temp))) return false;
+ if (UNLIKELY(!ToUnixTime(local_tz, &temp))) return false;
*unix_time = static_cast<double>(temp);
+
DCHECK(HasTime());
*unix_time += time_.fractional_seconds() * ONE_BILLIONTH;
return true;
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/service/frontend.cc
----------------------------------------------------------------------
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index 0108f63..592a1dd 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -99,7 +99,7 @@ Frontend::Frontend() {
{"loadTableData", "([B)[B", &load_table_data_id_},
{"getTableFiles", "([B)[B", &get_table_files_id_},
{"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_},
- {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_}
+ {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_},
};
JNIEnv* jni_env = getJNIEnv();
@@ -288,4 +288,4 @@ Status Frontend::GetTableFiles(const TShowFilesParams& params, TResultSet* resul
Status Frontend::BuildTestDescriptorTable(const TBuildTestDescriptorTableParams& params,
TDescriptorTable* result) {
return JniUtil::CallJniMethod(fe_, build_test_descriptor_table_id_, params, result);
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index a2d667c..c5d0308 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -45,6 +45,7 @@
#include "common/logging.h"
#include "common/version.h"
#include "exec/external-data-source-executor.h"
+#include "exprs/timezone_db.h"
#include "rpc/authentication.h"
#include "rpc/rpc-trace.h"
#include "rpc/thrift-thread.h"
@@ -976,10 +977,22 @@ void ImpalaServer::PrepareQueryContext(
const TNetworkAddress& backend_addr, TQueryCtx* query_ctx) {
query_ctx->__set_pid(getpid());
int64_t now_us = UnixMicros();
- query_ctx->__set_utc_timestamp_string(ToUtcStringFromUnixMicros(now_us));
- query_ctx->__set_now_string(ToStringFromUnixMicros(now_us));
+ const Timezone& utc_tz = TimezoneDatabase::GetUtcTimezone();
+ string local_tz_name = TimezoneDatabase::LocalZoneName();
+ const Timezone* local_tz = TimezoneDatabase::FindTimezone(local_tz_name);
+ if (local_tz != nullptr) {
+ LOG(INFO) << "Found local timezone \"" << local_tz_name << "\".";
+ } else {
+ LOG(ERROR) << "Failed to find local timezone \"" << local_tz_name
+ << "\". Falling back to UTC";
+ local_tz_name = "UTC";
+ local_tz = &utc_tz;
+ }
+ query_ctx->__set_utc_timestamp_string(ToStringFromUnixMicros(now_us, utc_tz));
+ query_ctx->__set_now_string(ToStringFromUnixMicros(now_us, *local_tz));
query_ctx->__set_start_unix_millis(now_us / MICROS_PER_MILLI);
query_ctx->__set_coord_address(backend_addr);
+ query_ctx->__set_local_time_zone(local_tz_name);
// Creating a random_generator every time is not free, but
// benchmarks show it to be slightly cheaper than contending for a
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/service/impalad-main.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impalad-main.cc b/be/src/service/impalad-main.cc
index b4ec3f0..2b7cc95 100644
--- a/be/src/service/impalad-main.cc
+++ b/be/src/service/impalad-main.cc
@@ -59,7 +59,6 @@ DECLARE_bool(is_coordinator);
int ImpaladMain(int argc, char** argv) {
InitCommonRuntime(argc, argv, true);
- ABORT_IF_ERROR(TimezoneDatabase::Initialize());
ABORT_IF_ERROR(LlvmCodeGen::InitializeLlvm());
JniUtil::InitLibhdfs();
ABORT_IF_ERROR(HBaseTableScanner::Init());
@@ -72,6 +71,13 @@ int ImpaladMain(int argc, char** argv) {
ExecEnv exec_env;
ABORT_IF_ERROR(exec_env.Init());
CommonMetrics::InitCommonMetrics(exec_env.metrics());
+
+ ABORT_IF_ERROR(TimezoneDatabase::Initialize());
+ // Add path to time-zone db as a property
+ StringProperty* tzdata_path = exec_env.metrics()->AddProperty<string>(
+ "tzdata-path", "");
+ tzdata_path->SetValue(TimezoneDatabase::GetPath());
+
ABORT_IF_ERROR(StartMemoryMaintenanceThread()); // Memory metrics are created in Init().
ABORT_IF_ERROR(
StartThreadInstrumentation(exec_env.metrics(), exec_env.webserver(), true));
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 99c8e6b..dd70b36 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -87,6 +87,7 @@ add_library(Util
url-parser.cc
${SQUEASEL_SRC_DIR}/squeasel.c
webserver.cc
+ zip-util.cc
${MUSTACHE_SRC_DIR}/mustache.cc
${MPFIT_SRC_DIR}/mpfit.c
)
@@ -144,3 +145,4 @@ ADD_BE_LSAN_TEST(thread-pool-test)
ADD_BE_LSAN_TEST(time-test)
ADD_BE_LSAN_TEST(uid-util-test)
ADD_BE_LSAN_TEST(webserver-test)
+ADD_BE_LSAN_TEST(zip-util-test)
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/filesystem-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/filesystem-util-test.cc b/be/src/util/filesystem-util-test.cc
index 5214483..be5f5eb 100644
--- a/be/src/util/filesystem-util-test.cc
+++ b/be/src/util/filesystem-util-test.cc
@@ -64,4 +64,58 @@ TEST(FilesystemUtil, RemoveAndCreateDirectory) {
filesystem::remove_all(dir);
}
+TEST(FilesystemUtil, Paths) {
+ // Canonical path must not be empty
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath(""));
+ // Canonical paths must be absolute
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("a/b"));
+ // Canonical paths must not contain "//", "..", "." components
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath(".."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/.."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a/b/.."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a/b/../c"));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a/b/."));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a/b/./c"));
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a//b"));
+ // Canonical paths must not end with '/'
+ EXPECT_FALSE(FileSystemUtil::IsCanonicalPath("/a/b/"));
+
+ // The following are valid canonical paths
+ EXPECT_TRUE(FileSystemUtil::IsCanonicalPath("/"));
+ EXPECT_TRUE(FileSystemUtil::IsCanonicalPath("/a"));
+ EXPECT_TRUE(FileSystemUtil::IsCanonicalPath("/ab/cd/efg"));
+
+ // The following should fail as "/a/b" is not a prefix of "/a/bc"
+ EXPECT_FALSE(FileSystemUtil::IsPrefixPath("/a/b", "/a/bc"));
+
+ // The following calls should succed.
+ EXPECT_TRUE(FileSystemUtil::IsPrefixPath("/", "/"));
+ EXPECT_TRUE(FileSystemUtil::IsPrefixPath("/", "/a/bc/def"));
+ EXPECT_TRUE(FileSystemUtil::IsPrefixPath("/a", "/a/bc/def"));
+ EXPECT_TRUE(FileSystemUtil::IsPrefixPath("/a/bc", "/a/bc/def"));
+ EXPECT_TRUE(FileSystemUtil::IsPrefixPath("/a/bc/def", "/a/bc/def"));
+
+ // 'relpath' should be empty if path equals to the start directory.
+ string relpath;
+ EXPECT_TRUE(FileSystemUtil::GetRelativePath("/", "/", &relpath));
+ EXPECT_EQ(string(""), relpath);
+ EXPECT_TRUE(FileSystemUtil::GetRelativePath("/a/bc/def", "/a/bc/def", &relpath));
+ EXPECT_EQ(string(""), relpath);
+
+ // The following should fail as "/a/b" is not a prefix of "/a/bc" path.
+ EXPECT_FALSE(FileSystemUtil::GetRelativePath("/a/bc", "/a/b", &relpath));
+
+ // The following calls should succeed.
+ EXPECT_TRUE(FileSystemUtil::GetRelativePath("/a/bc/def", "/", &relpath));
+ EXPECT_EQ(string("a/bc/def"), relpath);
+
+ EXPECT_TRUE(FileSystemUtil::GetRelativePath("/a/bc/def", "/a", &relpath));
+ EXPECT_EQ(string("bc/def"), relpath);
+
+ EXPECT_TRUE(FileSystemUtil::GetRelativePath("/a/bc/def", "/a/bc", &relpath));
+ EXPECT_EQ(string("def"), relpath);
+}
+
IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/filesystem-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/filesystem-util.cc b/be/src/util/filesystem-util.cc
index a0cacdf..83fce2f 100644
--- a/be/src/util/filesystem-util.cc
+++ b/be/src/util/filesystem-util.cc
@@ -21,6 +21,7 @@
#include <boost/filesystem.hpp>
#include <gutil/strings/substitute.h>
+#include "runtime/io/error-converter.h"
#include "util/filesystem-util.h"
#include "util/error-util.h"
@@ -158,4 +159,124 @@ uint64_t FileSystemUtil::MaxNumFileHandles() {
return 0ul;
}
+Status FileSystemUtil::GetCanonicalPath(const string& file_path, string* canonical_path) {
+ DCHECK(canonical_path != nullptr);
+ char rp[PATH_MAX];
+ if (realpath(file_path.c_str(), rp) == nullptr) {
+ return Status(ErrorMsg(TErrorCode::RUNTIME_ERROR,
+ Substitute("Resolving path for $0 failed with errno=$1 description=$2",
+ file_path.c_str(), errno, GetStrErrMsg())));
+ }
+ *canonical_path = rp;
+ return Status::OK();
+}
+
+bool FileSystemUtil::IsCanonicalPath(const string& path) {
+ return !path.empty()
+ && path.front() == '/'
+ && (path.length() == 1 || path.back() != '/')
+ && (path.length() == 1 || strcmp(path.c_str() + path.length() - 2, "/.") != 0)
+ && (path.length() <= 2 || strcmp(path.c_str() + path.length() - 3, "/..") != 0)
+ && strstr(path.c_str(), "//") == nullptr
+ && strstr(path.c_str(), "/./") == nullptr
+ && strstr(path.c_str(), "/../") == nullptr;
+}
+
+Status FileSystemUtil::IsSymbolicLink(const string& file_path, bool* is_symbolic_link,
+ string* canonical_path) {
+ DCHECK(is_symbolic_link != nullptr);
+ DCHECK(canonical_path != nullptr);
+ struct stat sb;
+ if (lstat(file_path.c_str(), &sb) == -1) {
+ return Status(ErrorMsg(TErrorCode::RUNTIME_ERROR,
+ Substitute("Getting file status for $0 failed with errno=$1 description=$2",
+ file_path.c_str(), errno, GetStrErrMsg())));
+ }
+
+ *is_symbolic_link = S_ISLNK(sb.st_mode) && sb.st_size > 0;
+ if (*is_symbolic_link) {
+ return GetCanonicalPath(file_path, canonical_path);
+ }
+ return Status::OK();
+}
+
+bool FileSystemUtil::IsPrefixPath(const string& prefix, const string& path) {
+ DCHECK(IsCanonicalPath(prefix));
+ DCHECK(IsCanonicalPath(path));
+ return !prefix.empty()
+ && (prefix == path
+ || (prefix.length() < path.length()
+ && strncmp(prefix.c_str(), path.c_str(), prefix.length()) == 0
+ && (prefix.back() == '/' || path[prefix.length()] == '/')));
+}
+
+bool FileSystemUtil::GetRelativePath(const string& path, const string& start,
+ string* relpath) {
+ DCHECK(IsCanonicalPath(path));
+ DCHECK(IsCanonicalPath(start));
+ DCHECK(relpath != nullptr);
+ if (IsPrefixPath(start, path)) {
+ *relpath = path.substr(start.length());
+ if (!relpath->empty() && relpath->front() == '/') *relpath = relpath->substr(1);
+ return true;
+ }
+ return false;
+}
+
+
+FileSystemUtil::Directory::Directory(const string& path)
+ : dir_path_(path),
+ status_(Status::OK()) {
+ dir_stream_ = opendir(dir_path_.c_str());
+ if (dir_stream_ == nullptr) {
+ status_ = ErrorConverter::GetErrorStatusFromErrno("opendir()", dir_path_, errno);
+ }
+}
+
+FileSystemUtil::Directory::~Directory() {
+ if (dir_stream_ != nullptr) (void)closedir(dir_stream_);
+}
+
+bool FileSystemUtil::Directory::GetNextEntryName(string* entry_name) {
+ DCHECK(entry_name != nullptr);
+
+ if (status_.ok()) {
+ DCHECK(dir_stream_ != nullptr);
+
+ errno = 0;
+ const dirent* dir_entry = nullptr;
+ while ((dir_entry = readdir(dir_stream_)) != nullptr) {
+ if (dir_entry->d_name[0] == 0 || strcmp(dir_entry->d_name, ".") == 0
+ || strcmp(dir_entry->d_name, "..") == 0) {
+ continue;
+ }
+ *entry_name = dir_entry->d_name;
+ return true;
+ }
+
+ // readdir() returned nullptr:
+ // Either readdir() failed or no more entries were found in 'dir_stream'.
+ if (errno != 0) {
+ status_ = ErrorConverter::GetErrorStatusFromErrno(
+ "readdir()", dir_path_, errno);
+ }
+ }
+ return false;
+}
+
+Status FileSystemUtil::Directory::GetEntryNames(const string& path,
+ vector<string>* entry_names, int max_result_size) {
+ DCHECK(entry_names != nullptr);
+
+ Directory dir(path);
+ entry_names->clear();
+ string entry_name;
+ while ((max_result_size <= 0 || entry_names->size() < max_result_size)
+ && dir.GetNextEntryName(&entry_name)) {
+ entry_names->push_back(entry_name);
+ }
+
+ return dir.GetLastStatus();
+}
+
} // namespace impala
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/filesystem-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/filesystem-util.h b/be/src/util/filesystem-util.h
index 1d497c3..c87fb9c 100644
--- a/be/src/util/filesystem-util.h
+++ b/be/src/util/filesystem-util.h
@@ -18,6 +18,7 @@
#ifndef IMPALA_UTIL_FILESYSTEM_UTIL_H
#define IMPALA_UTIL_FILESYSTEM_UTIL_H
+#include <dirent.h>
#include "common/status.h"
namespace impala {
@@ -52,6 +53,70 @@ class FileSystemUtil {
/// Returns the currently allowed maximum of possible file descriptors. In case of an
/// error returns 0.
static uint64_t MaxNumFileHandles();
+
+ /// Finds the canonicalized absolute pathname for 'file_path' and returns it in
+ /// *canonical_path.
+ static Status GetCanonicalPath(
+ const std::string& file_path, std::string* canonical_path) WARN_UNUSED_RESULT;
+
+ /// Checks if 'file_path' is a symbolic link. If it is, 'is_symbolic_link' is set to
+ /// 'true' and *canonical_path is set to the resolved canonicalized path.
+ static Status IsSymbolicLink(const std::string& file_path, bool* is_symbolic_link,
+ std::string* canonical_path) WARN_UNUSED_RESULT;
+
+ /// Returns 'true' iff 'path' is a canonicalized path. 'path' doesn't have to be an
+ /// existing path.
+ /// Always returns 'true' for the *canonical_path returned by GetCanonicalPath() and
+ /// IsSymbolicLink().
+ static bool IsCanonicalPath(const std::string& path);
+
+ /// Returns 'true' iff path 'prefix' is a non-empty prefix of path 'path'.
+ /// This is a string computation: the filesystem is not accessed to confirm the
+ /// existance of 'path' or 'prefix'. It is assumed that 'prefix' and 'path' are both
+ /// canonicalized paths.
+ static bool IsPrefixPath(const std::string& prefix, const std::string& path);
+
+ /// - If 'start' is a prefix of 'path', it constructs relative filepath to 'path' from
+ /// the 'start' directory and sets 'relpath' to the resulting path. 'true' is returned.
+ /// - Otherwise, 'relpath' is left intact and 'false' is returned.
+ /// This is a string computation: the filesystem is not accessed to confirm the
+ /// existance of 'path' or 'start'. It is assumed that 'path' and 'start' are both
+ /// canonicalized paths.
+ static bool GetRelativePath(const std::string& path, const std::string& start,
+ std::string* relpath);
+
+ class Directory {
+ public:
+ /// Opens 'path' directory for iteration. Directory entries "." and ".." will be
+ /// skipped while iterating through the entries.
+ Directory(const string& path);
+
+ /// Closes the directory.
+ ~Directory();
+
+ /// Reads the next directory entry and sets 'entry_name' to the entry name.
+ /// Returns 'false' if an error occured or no more entries were found in the
+ /// directory. Return 'true' on success.
+ bool GetNextEntryName(std::string* entry_name);
+
+ /// Returns the status of the previous directory operation.
+ const Status& GetLastStatus() const { return status_; }
+
+ /// Reads no more than 'max_result_size' directory entries from 'path' and returns
+ /// their names in 'entry_names' vector. If 'max_result_size' <= 0, every directory
+ /// entry is returned. Directory entries "." and ".." will be skipped.
+ static Status GetEntryNames(const string& path, std::vector<std::string>* entry_names,
+ int max_result_size = 0);
+
+ private:
+ DIR* dir_stream_;
+ std::string dir_path_;
+ Status status_;
+
+ // Do not allow making copies.
+ Directory(const Directory&);
+ Directory& operator=(const Directory&);
+ };
};
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/hdfs-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/hdfs-util-test.cc b/be/src/util/hdfs-util-test.cc
index bf8c643..7ed3646 100644
--- a/be/src/util/hdfs-util-test.cc
+++ b/be/src/util/hdfs-util-test.cc
@@ -74,6 +74,44 @@ TEST(HdfsUtilTest, CheckFilesystemsMatch) {
"temp_dir/temp_path"));
}
+TEST(HdfsUtilTest, CheckGetBaseName) {
+ // We do not want to start the webserver.
+ FLAGS_enable_webserver = false;
+
+ EXPECT_EQ(".", GetBaseName("s3a://"));
+ EXPECT_EQ(".", GetBaseName("s3a://dummybucket"));
+ EXPECT_EQ(".", GetBaseName("s3a://dummybucket/"));
+ EXPECT_EQ(".", GetBaseName("s3a://dummybucket//"));
+ EXPECT_EQ("c", GetBaseName("s3a://dummybucket/c/"));
+ EXPECT_EQ("c", GetBaseName("s3a://dummybucket/a/b/c"));
+ EXPECT_EQ("c", GetBaseName("s3a://dummybucket/a/b/c///"));
+
+ EXPECT_EQ(".", GetBaseName("s3a:///"));
+ EXPECT_EQ("c", GetBaseName("s3a:///c"));
+ EXPECT_EQ("c", GetBaseName("s3a:///a/b//c/"));
+
+ EXPECT_EQ(".", GetBaseName("hdfs://"));
+ EXPECT_EQ(".", GetBaseName("hdfs://localhost"));
+ EXPECT_EQ(".", GetBaseName("hdfs://localhost:8020"));
+ EXPECT_EQ(".", GetBaseName("hdfs://localhost:8020/"));
+ EXPECT_EQ(".", GetBaseName("hdfs://localhost:8020//"));
+ EXPECT_EQ("c", GetBaseName("hdfs://localhost:8020/c"));
+ EXPECT_EQ("c", GetBaseName("hdfs://localhost:8020/c/"));
+ EXPECT_EQ("c", GetBaseName("hdfs://localhost:8020/a//b/c"));
+ EXPECT_EQ("c", GetBaseName("hdfs://localhost:8020//a//b/c/"));
+ EXPECT_EQ("c", GetBaseName("hdfs://localhost:8020//a//b/c///"));
+
+ EXPECT_EQ(".", GetBaseName("hdfs:///"));
+ EXPECT_EQ("c", GetBaseName("hdfs:///c"));
+ EXPECT_EQ("c", GetBaseName("hdfs:///a/b//c/"));
+
+ EXPECT_EQ(".", GetBaseName(""));
+ EXPECT_EQ(".", GetBaseName("/"));
+ EXPECT_EQ("c", GetBaseName("c"));
+ EXPECT_EQ("c", GetBaseName("/c"));
+ EXPECT_EQ("c", GetBaseName("/a//b/c/"));
+}
+
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
InitCommonRuntime(argc, argv, true, TestInfo::BE_TEST);
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/hdfs-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/hdfs-util.cc b/be/src/util/hdfs-util.cc
index 28d318c..aa3466f 100644
--- a/be/src/util/hdfs-util.cc
+++ b/be/src/util/hdfs-util.cc
@@ -24,6 +24,7 @@
#include "common/names.h"
#include "runtime/exec-env.h"
+#include "kudu/util/path_util.h"
namespace impala {
@@ -133,4 +134,14 @@ bool FilesystemsMatch(const char* path_a, const char* path_b) {
return strncmp(path_a, path_b, fs_a_name_length) == 0;
}
+string GetBaseName(const char* path) {
+ int fs_name_length = GetFilesystemNameLength(path);
+ if (fs_name_length >= strlen(path)) return ".";
+
+ string bname = kudu::BaseName(&(path[fs_name_length]));
+ if (bname.empty() || bname == "/") return ".";
+
+ return bname;
+}
+
}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/hdfs-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/hdfs-util.h b/be/src/util/hdfs-util.h
index b9f415b..858a9b4 100644
--- a/be/src/util/hdfs-util.h
+++ b/be/src/util/hdfs-util.h
@@ -55,5 +55,10 @@ bool IsADLSPath(const char* path);
/// Returns true iff 'pathA' and 'pathB' are on the same filesystem.
bool FilesystemsMatch(const char* pathA, const char* pathB);
+
+/// Returns the terminal component of 'path'.
+/// E.g. if 'path' is "hdfs://localhost:8020/a/b/c", "c" is returned.
+/// If the terminal component is empty string or "/", the function returns ".".
+string GetBaseName(const char* path);
}
#endif // IMPALA_UTIL_HDFS_UTIL_H
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/time-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/time-test.cc b/be/src/util/time-test.cc
index dde0d12..78b7958 100644
--- a/be/src/util/time-test.cc
+++ b/be/src/util/time-test.cc
@@ -22,6 +22,7 @@
#include "util/time.h"
#include "testutil/gtest-util.h"
+#include "exprs/timezone_db.h"
#include "common/names.h"
using namespace std;
@@ -29,6 +30,7 @@ using namespace std;
namespace impala {
// Basic tests for the time formatting APIs in util/time.h.
TEST(TimeTest, Basic) {
+ const Timezone& utc_tz = cctz::utc_time_zone();
EXPECT_EQ("1970-01-01 00:00:00", ToUtcStringFromUnix(0));
EXPECT_EQ("1970-01-01 00:00:00.000", ToUtcStringFromUnixMillis(0));
EXPECT_EQ("1970-01-01 00:00:00.001", ToUtcStringFromUnixMillis(1));
@@ -38,6 +40,16 @@ TEST(TimeTest, Basic) {
EXPECT_EQ("1970-01-01 00:00:00.000", ToUtcStringFromUnixMicros(0,
TimePrecision::Millisecond));
EXPECT_EQ("1970-01-01 00:00:00", ToUtcStringFromUnixMicros(0, TimePrecision::Second));
+ EXPECT_EQ("1970-01-01 00:00:00.000000", ToStringFromUnixMicros(0, utc_tz));
+ EXPECT_EQ("1970-01-01 00:00:00.000001", ToStringFromUnixMicros(1, utc_tz));
+ EXPECT_EQ("1970-01-01 00:00:00.000", ToStringFromUnixMicros(0, utc_tz,
+ TimePrecision::Millisecond));
+ EXPECT_EQ("1970-01-01 00:00:00.000", ToStringFromUnixMicros(999, utc_tz,
+ TimePrecision::Millisecond));
+ EXPECT_EQ("1970-01-01 00:00:00", ToStringFromUnixMicros(0, utc_tz,
+ TimePrecision::Second));
+ EXPECT_EQ("1970-01-01 00:00:00", ToStringFromUnixMicros(999999, utc_tz,
+ TimePrecision::Second));
EXPECT_EQ("1970-01-01 00:01:00", ToUtcStringFromUnix(60));
EXPECT_EQ("1970-01-01 01:00:00", ToUtcStringFromUnix(3600));
@@ -49,10 +61,12 @@ TEST(TimeTest, Basic) {
// -9223372036854775808 (INT64_MIN) nanoseconds before the Unix epoch.
EXPECT_EQ("1677-09-21 00:12:44",
ToUtcStringFromUnix(INT64_MIN / NANOS_PER_SEC));
- EXPECT_EQ("1677-09-21 00:12:44.854",
+ EXPECT_EQ("1677-09-21 00:12:43.146",
ToUtcStringFromUnixMillis(INT64_MIN / NANOS_PER_MICRO / MICROS_PER_MILLI));
- EXPECT_EQ("1677-09-21 00:12:44.854775",
+ EXPECT_EQ("1677-09-21 00:12:43.145225",
ToUtcStringFromUnixMicros(INT64_MIN / NANOS_PER_MICRO));
+ EXPECT_EQ("1677-09-21 00:12:43.145225",
+ ToStringFromUnixMicros(INT64_MIN / NANOS_PER_MICRO, utc_tz));
// The latest date-time that can be represented with the high-resoliution
// chrono::system_clock is 2262-04-11 23:47:16 UTC, which is
@@ -63,15 +77,24 @@ TEST(TimeTest, Basic) {
ToUtcStringFromUnixMillis(INT64_MAX / NANOS_PER_MICRO / MICROS_PER_MILLI));
EXPECT_EQ("2262-04-11 23:47:16.854775",
ToUtcStringFromUnixMicros(INT64_MAX / NANOS_PER_MICRO));
+ EXPECT_EQ("2262-04-11 23:47:16.854775",
+ ToStringFromUnixMicros(INT64_MAX / NANOS_PER_MICRO, utc_tz));
+ EXPECT_EQ("1969-12-31 23:59:59.999", ToUtcStringFromUnixMillis(-1));
+ EXPECT_EQ("1969-12-31 23:59:59.001", ToUtcStringFromUnixMillis(-999));
EXPECT_EQ("1969-12-31 23:59:59.000", ToUtcStringFromUnixMillis(-1000));
- EXPECT_EQ("1969-12-31 23:59:59.999", ToUtcStringFromUnixMillis(-1999));
+ EXPECT_EQ("1969-12-31 23:59:58.001", ToUtcStringFromUnixMillis(-1999));
EXPECT_EQ("1969-12-31 23:59:58.000", ToUtcStringFromUnixMillis(-2000));
- EXPECT_EQ("1969-12-31 23:59:58.001", ToUtcStringFromUnixMillis(-2001));
+ EXPECT_EQ("1969-12-31 23:59:57.999", ToUtcStringFromUnixMillis(-2001));
+ EXPECT_EQ("1969-12-31 23:59:59.999999", ToUtcStringFromUnixMicros(-1));
+ EXPECT_EQ("1969-12-31 23:59:59.000001", ToUtcStringFromUnixMicros(-999999));
EXPECT_EQ("1969-12-31 23:59:59.000000", ToUtcStringFromUnixMicros(-1000000));
- EXPECT_EQ("1969-12-31 23:59:59.999999", ToUtcStringFromUnixMicros(-1999999));
- EXPECT_EQ("1969-12-31 23:59:58.000001", ToUtcStringFromUnixMicros(-2000001));
+ EXPECT_EQ("1969-12-31 23:59:58.000001", ToUtcStringFromUnixMicros(-1999999));
+ EXPECT_EQ("1969-12-31 23:59:57.999999", ToUtcStringFromUnixMicros(-2000001));
+ EXPECT_EQ("1969-12-31 23:59:59.000000", ToStringFromUnixMicros(-1000000, utc_tz));
+ EXPECT_EQ("1969-12-31 23:59:58.000001", ToStringFromUnixMicros(-1999999, utc_tz));
+ EXPECT_EQ("1969-12-31 23:59:57.999999", ToStringFromUnixMicros(-2000001, utc_tz));
// Unix time does not represent leap seconds. Test continuous roll-over of
// Unix time after 1998-12-31 23:59:59
@@ -105,8 +128,10 @@ TEST(TimeTest, Basic) {
int64_t now_us = UnixMicros();
s1 = ToStringFromUnixMicros(now_us, TimePrecision::Nanosecond);
s2 = ToUtcStringFromUnixMicros(now_us, TimePrecision::Nanosecond);
+ string s3 = ToStringFromUnixMicros(now_us, utc_tz, TimePrecision::Nanosecond);
EXPECT_TRUE(regex_search(s1, sm, regex(R"(\.\d{6}(000)$)")));
EXPECT_TRUE(regex_search(s2, sm, regex(R"(\.\d{6}(000)$)")));
+ EXPECT_TRUE(regex_search(s3, sm, regex(R"(\.\d{6}(000)$)")));
} // TEST
} // namespace impala
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/time.cc
----------------------------------------------------------------------
diff --git a/be/src/util/time.cc b/be/src/util/time.cc
index dd0587b..f6adcdd 100644
--- a/be/src/util/time.cc
+++ b/be/src/util/time.cc
@@ -21,6 +21,7 @@
#include <sstream>
#include <cstdlib>
+#include "exprs/timezone_db.h"
#include "util/time.h"
using namespace impala;
@@ -30,14 +31,36 @@ void impala::SleepForMs(const int64_t duration_ms) {
this_thread::sleep_for(chrono::milliseconds(duration_ms));
}
-// Convert the given time_point, 't', into a date-time string in the
+static inline time_t GetSecond(const chrono::system_clock::time_point& t) {
+ return chrono::system_clock::to_time_t(t);
+}
+
+static inline int64_t GetSubSecond(const chrono::system_clock::time_point& t,
+ TimePrecision p) {
+ auto frac = t.time_since_epoch();
+ if (p == TimePrecision::Millisecond) {
+ auto subsec = chrono::duration_cast<chrono::milliseconds>(frac) % MILLIS_PER_SEC;
+ return subsec.count();
+ } else if (p == TimePrecision::Microsecond) {
+ auto subsec = chrono::duration_cast<chrono::microseconds>(frac) % MICROS_PER_SEC;
+ return subsec.count();
+ } else if (p == TimePrecision::Nanosecond) {
+ auto subsec = chrono::duration_cast<chrono::nanoseconds>(frac) % NANOS_PER_SEC;
+ return subsec.count();
+ } else {
+ return 0;
+ }
+}
+
+// Convert the given 'second' unix timestamp, into a date-time string in the
// UTC time zone if 'utc' is true, or the local time zone if it is false.
// The returned string is of the form yyy-MM-dd HH::mm::SS.
-static string TimepointToString(const chrono::system_clock::time_point& t,
- bool utc) {
+// Note that for time points before the Unix epoch, 'subsecond' might have a negative
+// value. In this case 'second' has to be adjusted.
+static string FormatSecond(time_t second, int64_t subsecond, bool utc) {
char buf[256];
struct tm tmp;
- auto input_time = chrono::system_clock::to_time_t(t);
+ auto input_time = (subsecond < 0) ? second - 1 : second;
// gcc 4.9 does not support C++14 get_time and put_time functions, so we're
// stuck with strftime() for now.
@@ -49,24 +72,21 @@ static string TimepointToString(const chrono::system_clock::time_point& t,
return string(buf);
}
-// Format the sub-second part of the input time point object 't', at the
-// precision specified by 'p'. The returned string is meant to be appended to
-// the string returned by TimePointToString() above.
-// Note the use of abs(). This is to make sure we correctly format negative times,
-// i.e., times before the Unix epoch.
-static string FormatSubSecond(const chrono::system_clock::time_point& t,
- TimePrecision p) {
+// Format the sub-second part of a time point, at the precision specified by 'p'. The
+// returned string is meant to be appended to the string returned by FormatSecond()
+// above. Note that for time points before the Unix epoch 'subsecond' might have a
+// negative value. In this case 'subsecond' has to be adjusted based on the precision.
+static string FormatSubSecond(int64_t subsecond, TimePrecision p) {
stringstream ss;
- auto frac = t.time_since_epoch();
if (p == TimePrecision::Millisecond) {
- auto subsec = chrono::duration_cast<chrono::milliseconds>(frac) % MILLIS_PER_SEC;
- ss << "." << std::setfill('0') << std::setw(3) << abs(subsec.count());
+ if (subsecond < 0) subsecond = 1000 + subsecond;
+ ss << "." << std::setfill('0') << std::setw(3) << subsecond;
} else if (p == TimePrecision::Microsecond) {
- auto subsec = chrono::duration_cast<chrono::microseconds>(frac) % MICROS_PER_SEC;
- ss << "." << std::setfill('0') << std::setw(6) << abs(subsec.count());
+ if (subsecond < 0) subsecond = 1000000 + subsecond;
+ ss << "." << std::setfill('0') << std::setw(6) << subsecond;
} else if (p == TimePrecision::Nanosecond) {
- auto subsec = chrono::duration_cast<chrono::nanoseconds>(frac) % NANOS_PER_SEC;
- ss << "." << std::setfill('0') << std::setw(9) << abs(subsec.count());
+ if (subsecond < 0) subsecond = 1000000000 + subsecond;
+ ss << "." << std::setfill('0') << std::setw(9) << subsecond;
} else {
// 1-second precision or unknown unit. Return empty string.
DCHECK_EQ(TimePrecision::Second, p);
@@ -79,11 +99,12 @@ static string FormatSubSecond(const chrono::system_clock::time_point& t,
// Output string is in UTC time zone if 'utc' is true, else it is in the
// local time zone.
static string ToString(const chrono::system_clock::time_point& t, TimePrecision p,
- bool utc)
-{
+ bool utc) {
stringstream ss;
- ss << TimepointToString(t, utc);
- ss << FormatSubSecond(t, p);
+ time_t second = GetSecond(t);
+ int64_t subsecond = GetSubSecond(t, p);
+ ss << FormatSecond(second, subsecond, utc);
+ ss << FormatSubSecond(subsecond, p);
return ss.str();
}
@@ -134,3 +155,23 @@ string impala::ToUtcStringFromUnixMicros(int64_t us, TimePrecision p) {
chrono::system_clock::time_point t = TimepointFromUnixMicros(us);
return ToString(t, p, true);
}
+
+string impala::ToStringFromUnixMicros(int64_t us, const Timezone& tz,
+ TimePrecision p) {
+ chrono::system_clock::time_point t = TimepointFromUnixMicros(us);
+ const char* fmt;
+ switch (p) {
+ case TimePrecision::Millisecond:
+ fmt = "%Y-%m-%d %H:%M:%E3S";
+ break;
+ case TimePrecision::Microsecond:
+ fmt = "%Y-%m-%d %H:%M:%E6S";
+ break;
+ case TimePrecision::Nanosecond:
+ fmt = "%Y-%m-%d %H:%M:%E9S";
+ break;
+ default:
+ fmt = "%Y-%m-%d %H:%M:%S";
+ }
+ return cctz::format(fmt, t, tz);
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/time.h
----------------------------------------------------------------------
diff --git a/be/src/util/time.h b/be/src/util/time.h
index 64dbf9c..f19d21b 100644
--- a/be/src/util/time.h
+++ b/be/src/util/time.h
@@ -22,6 +22,7 @@
#include <time.h>
#include <string>
+#include "common/global-types.h"
#include "gutil/walltime.h"
/// Utilities for collecting timings.
@@ -116,6 +117,11 @@ std::string ToStringFromUnixMicros(int64_t us,
std::string ToUtcStringFromUnixMicros(int64_t us,
TimePrecision p = TimePrecision::Microsecond);
+/// Converts input microseconds-since-epoch to date-time string in 'tz' time zone.
+/// In the returned string fractional seconds are padded to precision 'p'.
+std::string ToStringFromUnixMicros(int64_t us, const Timezone& tz,
+ TimePrecision p = TimePrecision::Microsecond);
+
/// Convenience function to convert the current time, derived from UnixMicros(),
/// to a date-time string in the local time zone, padded to nanosecond precision.
inline std::string CurrentTimeString() {
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/zip-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/zip-util-test.cc b/be/src/util/zip-util-test.cc
new file mode 100644
index 0000000..c3379f6
--- /dev/null
+++ b/be/src/util/zip-util-test.cc
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/zip-util.h"
+
+#include <boost/filesystem.hpp>
+#include <gtest/gtest.h>
+#include <sys/stat.h>
+
+#include "common/init.h"
+#include "common/status.h"
+#include "testutil/gtest-util.h"
+#include "util/filesystem-util.h"
+#include "util/test-info.h"
+
+#include "common/names.h"
+
+namespace filesystem = boost::filesystem;
+using filesystem::path;
+
+using namespace impala;
+
+// Tests for util/zip-util.cc.
+TEST(ZipUtilTest, Basic) {
+ // Create path to a unique directory to hold destination directories.
+ path uniq_dir = filesystem::unique_path();
+ // Make sure the directory doesn't exist yet.
+ EXPECT_FALSE(filesystem::exists(uniq_dir));
+
+ // Extracting files from a valid zip archive should be successful.
+ string archive_file1 = Substitute("$0/testdata/tzdb/2017c.zip", getenv("IMPALA_HOME"));
+ path dest_dir1 = uniq_dir / "impala.zip.test.1" / "2017c";
+ EXPECT_OK(ZipUtil::ExtractFiles(archive_file1, dest_dir1.string()));
+ // Verify that some extracted files exist.
+ path zone_info_root_dir = dest_dir1 / "zoneinfo";
+ EXPECT_TRUE(filesystem::exists(zone_info_root_dir / "UTC"));
+ EXPECT_TRUE(filesystem::exists(zone_info_root_dir / "America" / "Indiana" / "Marengo"));
+
+ // Extracting files from a non-existent zip archive should fail.
+ string archive_file2 = Substitute("$0/testdata/tzdb/non-existent.zip",
+ getenv("IMPALA_HOME"));
+ path dest_dir2 = uniq_dir / "impala.zip.test.2";
+ EXPECT_ERROR(ZipUtil::ExtractFiles(archive_file2, dest_dir2.string()),
+ TErrorCode::GENERAL);
+ // Make sure destination directory hasn't been created.
+ EXPECT_FALSE(filesystem::exists(dest_dir2));
+
+ // Extracting files from a corrupt zip archive should fail.
+ string archive_file3 = Substitute("$0/testdata/tzdb/2017c-corrupt.zip",
+ getenv("IMPALA_HOME"));
+ path dest_dir3 = uniq_dir / "impala.zip.test.3";
+ EXPECT_ERROR(ZipUtil::ExtractFiles(archive_file3, dest_dir3.string()),
+ TErrorCode::GENERAL);
+
+ // Extracting files to a non-writable directory should fail.
+ string archive_file4 = Substitute("$0/testdata/tzdb/2017c.zip", getenv("IMPALA_HOME"));
+ path dest_dir4 = uniq_dir / "impala.zip.test.4";
+ EXPECT_TRUE(mkdir(dest_dir4.string().c_str(),
+ S_IRUSR | S_IXUSR | S_IRWXG | S_IRWXO) == 0);
+ EXPECT_ERROR(ZipUtil::ExtractFiles(archive_file4, dest_dir4.string()),
+ TErrorCode::GENERAL);
+ // Make sure the archive's root directory hasn't been created.
+ EXPECT_FALSE(filesystem::exists(dest_dir4 / "zoneinfo"));
+
+ // Cleanup
+ filesystem::remove_all(uniq_dir);
+}
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ InitCommonRuntime(argc, argv, true, TestInfo::BE_TEST);
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/zip-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/zip-util.cc b/be/src/util/zip-util.cc
new file mode 100644
index 0000000..b8777fa
--- /dev/null
+++ b/be/src/util/zip-util.cc
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/zip-util.h"
+
+#include "gen-cpp/Zip_types.h"
+#include "rpc/jni-thrift-util.h"
+
+#include "common/names.h"
+
+namespace impala {
+
+jclass ZipUtil::zip_util_class_;
+jmethodID ZipUtil::extract_files_method; // ZipUtil.extractFiles()
+
+void ZipUtil::InitJvm() {
+ JNIEnv* env = getJNIEnv();
+ zip_util_class_ = env->FindClass("org/apache/impala/util/ZipUtil");
+ EXIT_IF_EXC(env);
+ JniMethodDescriptor extract_files_method_desc =
+ {"extractFiles", "([B)V", &extract_files_method};
+
+ ABORT_IF_ERROR(JniUtil::LoadStaticJniMethod(
+ env, zip_util_class_, &extract_files_method_desc));
+}
+
+Status ZipUtil::ExtractFiles(const string& archive_file, const string& destination_dir) {
+ TExtractFromZipParams params;
+ params.archive_file = archive_file;
+ params.destination_dir = destination_dir;
+ return JniUtil::CallJniMethod(zip_util_class_, extract_files_method, params);
+}
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/be/src/util/zip-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/zip-util.h b/be/src/util/zip-util.h
new file mode 100644
index 0000000..3674ba9
--- /dev/null
+++ b/be/src/util/zip-util.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef IMPALA_UTIL_ZIP_UTIL_H
+#define IMPALA_UTIL_ZIP_UTIL_H
+
+#include <string>
+
+#include "util/jni-util.h"
+#include "common/status.h"
+
+namespace impala {
+
+class ZipUtil {
+ public:
+ /// Loads the JNI helper method to extract files from a Zip archive.
+ static void InitJvm();
+
+ /// Extract files from a zip archive to a destination directory in local filesystem.
+ static Status ExtractFiles(const std::string& archive_file,
+ const std::string& destination_dir) WARN_UNUSED_RESULT;
+
+ private:
+ static jclass zip_util_class_;
+ static jmethodID extract_files_method;
+};
+
+}
+#endif
http://git-wip-us.apache.org/repos/asf/impala/blob/17749dbc/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 6070350..42b59fb 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -421,7 +421,7 @@ if __name__ == "__main__":
# LLVM and Kudu are the largest packages. Sort them first so that
# their download starts as soon as possible.
packages = map(Package, ["llvm", "kudu",
- "avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "crcutil",
+ "avro", "binutils", "boost", "breakpad", "bzip2", "cctz", "cmake", "crcutil",
"flatbuffers", "gcc", "gflags", "glog", "gperftools", "gtest", "libev",
"lz4", "openldap", "openssl", "orc", "protobuf",
"rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"])