You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/09/30 07:26:02 UTC
[5/9] impala git commit: IMPALA-7492: Add support for DATE text
parser/formatter
IMPALA-7492: Add support for DATE text parser/formatter
This change is the first step in implementing support for DATE type
(IMPALA-6169).
The DATE parser/formatter is implemented by the new DateParser class.
- The parser supports parsing both default and custom formatted DATE
values. CCTZ is used to validate the parsed dates.
- The formatter supports default and custom formatting of DATE values.
In the future, DateParser will be used in the text scanner/writer and
in the DATE <-> STRING cast functions.
The DateParser class reuses some of the functionality already
implemented in the TimestampParser class to minimize redundancy. To
make code reuse easier, a new namespace (datetime_parse_util) was
created and the common functionality was moved there.
This change also adds a new class (DateValue) to represent a DATE
value in-memory. The DateParser and DateValue classes are used only in
tests at the moment, therefore this patch doesn't change user facing
behavior.
Testing:
- Added BE-tests for DateParser and DateValue classes.
- Re-run parse-timestamp-benchmark to make sure that parser
performance hasn't degraded.
Change-Id: I1eec00f22502c4c67c6807c4b51384419ea8b831
Reviewed-on: http://gerrit.cloudera.org:8080/11450
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/cb493716
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/cb493716
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/cb493716
Branch: refs/heads/master
Commit: cb49371613909e56debee6275fd54759eb36ad33
Parents: 34e666f
Author: Attila Jeges <at...@cloudera.com>
Authored: Mon Sep 3 19:07:36 2018 +0200
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Sep 28 18:14:56 2018 +0000
----------------------------------------------------------------------
.../benchmarks/convert-timestamp-benchmark.cc | 12 +-
be/src/benchmarks/parse-timestamp-benchmark.cc | 10 +-
be/src/common/init.cc | 4 +-
be/src/exprs/timestamp-functions-ir.cc | 9 +-
be/src/exprs/timestamp-functions.cc | 7 +-
be/src/exprs/timestamp-functions.h | 8 +-
be/src/runtime/CMakeLists.txt | 4 +
be/src/runtime/date-parse-util.cc | 187 +++++++
be/src/runtime/date-parse-util.h | 69 +++
be/src/runtime/date-test.cc | 553 +++++++++++++++++++
be/src/runtime/date-value.cc | 119 ++++
be/src/runtime/date-value.h | 129 +++++
be/src/runtime/datetime-parse-util.cc | 516 +++++++++++++++++
be/src/runtime/datetime-parse-util.h | 246 +++++++++
be/src/runtime/timestamp-parse-util.cc | 494 +----------------
be/src/runtime/timestamp-parse-util.h | 231 +-------
be/src/runtime/timestamp-test.cc | 21 +-
be/src/runtime/timestamp-value.cc | 2 +
be/src/runtime/timestamp-value.h | 7 +-
19 files changed, 1896 insertions(+), 732 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/benchmarks/convert-timestamp-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/convert-timestamp-benchmark.cc b/be/src/benchmarks/convert-timestamp-benchmark.cc
index 235e3d9..89ba857 100644
--- a/be/src/benchmarks/convert-timestamp-benchmark.cc
+++ b/be/src/benchmarks/convert-timestamp-benchmark.cc
@@ -36,7 +36,7 @@
#include "cctz/civil_time.h"
#include "exprs/timezone_db.h"
#include "exprs/timestamp-functions.h"
-#include "runtime/timestamp-parse-util.h"
+#include "runtime/datetime-parse-util.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
#include "util/benchmark.h"
@@ -53,6 +53,10 @@ using std::thread;
using namespace impala;
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::InitParseCtx;
+using datetime_parse_util::ParseFormatTokens;
+
// Benchmark tests for timestamp time-zone conversions
/*
Machine Info: Intel(R) Core(TM) i5-6600 CPU @ 3.30GHz
@@ -145,8 +149,8 @@ fast path speedup: 10.2951
vector<TimestampValue> AddTestDataDateTimes(int n, const string& startstr) {
DateTimeFormatContext dt_ctx;
- dt_ctx.Reset("yyyy-MMM-dd HH:mm:ss", 19);
- TimestampParser::ParseFormatTokens(&dt_ctx);
+ dt_ctx.Reset("yyyy-MMM-dd HH:mm:ss");
+ ParseFormatTokens(&dt_ctx);
random_device rd;
mt19937 gen(rd());
@@ -621,7 +625,7 @@ int main(int argc, char* argv[]) {
PTR_CCTZ_LOCAL_TZ = TimezoneDatabase::FindTimezone(local_tz_name);
DCHECK(PTR_CCTZ_LOCAL_TZ != nullptr);
- TimestampParser::Init();
+ InitParseCtx();
const vector<TimestampValue> tsvalue_data =
AddTestDataDateTimes(1000, "1953-04-22 01:02:03");
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/benchmarks/parse-timestamp-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/parse-timestamp-benchmark.cc b/be/src/benchmarks/parse-timestamp-benchmark.cc
index 7aad0e6..d7bb34e 100644
--- a/be/src/benchmarks/parse-timestamp-benchmark.cc
+++ b/be/src/benchmarks/parse-timestamp-benchmark.cc
@@ -39,6 +39,10 @@ using boost::posix_time::to_iso_extended_string;
using boost::posix_time::to_simple_string;
using namespace impala;
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::InitParseCtx;
+using datetime_parse_util::ParseFormatTokens;
+
// Benchmark for parsing timestamps.
// Machine Info: Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz
// ParseDate: Function Rate (iters/ms) Comparison
@@ -202,7 +206,7 @@ int main(int argc, char **argv) {
CpuInfo::Init();
cout << Benchmark::GetMachineInfo() << endl;
- TimestampParser::Init();
+ InitParseCtx();
TestData dates, times, datetimes, tzdatetimes;
@@ -226,9 +230,9 @@ int main(int argc, char **argv) {
timestamp_suite.AddBenchmark("Impala", TestImpalaDate, ×);
dt_ctx.Reset("yyyy-MM-dd HH:mm:ss", 19);
- TimestampParser::ParseFormatTokens(&dt_ctx);
+ ParseFormatTokens(&dt_ctx);
dt_ctx_tz.Reset("yyyy-MM-dd HH:mm:ss+hh:mm", 25);
- TimestampParser::ParseFormatTokens(&dt_ctx_tz);
+ ParseFormatTokens(&dt_ctx_tz);
Benchmark timestamp_with_format_suite("ParseTimestampWithFormat");
timestamp_with_format_suite.AddBenchmark("BoostDateTime",
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index ed61fe4..0063667 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -31,12 +31,12 @@
#include "rpc/authentication.h"
#include "rpc/thrift-util.h"
#include "runtime/bufferpool/buffer-pool.h"
+#include "runtime/datetime-parse-util.h"
#include "runtime/decimal-value.h"
#include "runtime/exec-env.h"
#include "runtime/hdfs-fs-cache.h"
#include "runtime/lib-cache.h"
#include "runtime/mem-tracker.h"
-#include "runtime/timestamp-parse-util.h"
#include "util/cpu-info.h"
#include "util/debug-util.h"
#include "util/decimal-util.h"
@@ -220,7 +220,7 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
AtomicOps_x86CPUFeaturesInit();
#endif
impala::InitThreading();
- impala::TimestampParser::Init();
+ impala::datetime_parse_util::InitParseCtx();
impala::SeedOpenSSLRNG();
ABORT_IF_ERROR(impala::InitAuth(argv[0]));
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index c12c320..5b1dcfa 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -24,9 +24,9 @@
#include <gutil/strings/substitute.h>
#include "exprs/anyval-util.h"
+#include "runtime/datetime-parse-util.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-value.inline.h"
-#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
#include "udf/udf.h"
#include "udf/udf-internal.h"
@@ -56,13 +56,16 @@ typedef boost::posix_time::seconds Seconds;
namespace impala {
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::ParseFormatTokens;
+
StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
const TimestampValue& tv, const StringVal& fmt) {
void* state = context->GetFunctionState(FunctionContext::THREAD_LOCAL);
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!TimestampParser::ParseFormatTokens(dt_ctx)){
+ if (!ParseFormatTokens(dt_ctx)){
TimestampFunctions::ReportBadFormat(context, fmt, false);
return StringVal::null();
}
@@ -165,7 +168,7 @@ TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!TimestampParser::ParseFormatTokens(dt_ctx)) {
+ if (!ParseFormatTokens(dt_ctx)) {
ReportBadFormat(context, fmt, false);
return TimestampVal::null();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/exprs/timestamp-functions.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
index efc83cc..121b69d 100644
--- a/be/src/exprs/timestamp-functions.cc
+++ b/be/src/exprs/timestamp-functions.cc
@@ -21,8 +21,8 @@
#include "exprs/anyval-util.h"
#include "exprs/timezone_db.h"
#include "gutil/strings/substitute.h"
+#include "runtime/datetime-parse-util.h"
#include "runtime/string-value.inline.h"
-#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
#include "udf/udf-internal.h"
@@ -32,6 +32,9 @@
namespace impala {
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::ParseFormatTokens;
+
const string TimestampFunctions::DAY_ARRAY[7] = {"Sun", "Mon", "Tue", "Wed", "Thu",
"Fri", "Sat"};
const string TimestampFunctions::DAYNAME_ARRAY[7] = {"Sunday", "Monday", "Tuesday",
@@ -120,7 +123,7 @@ void TimestampFunctions::UnixAndFromUnixPrepare(
return;
}
dt_ctx = new DateTimeFormatContext(fmt_ref.ptr, fmt_ref.len);
- bool parse_result = TimestampParser::ParseFormatTokens(dt_ctx);
+ bool parse_result = ParseFormatTokens(dt_ctx);
if (!parse_result) {
delete dt_ctx;
TimestampFunctions::ReportBadFormat(context, fmt_val, true);
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/exprs/timestamp-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index ee0d54f..aa4c9b7 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -65,6 +65,10 @@ class TimestampFunctions {
static const int64_t MAX_MILLI_INTERVAL = MAX_SEC_INTERVAL * 1000;
static const int64_t MAX_MICRO_INTERVAL = MAX_MILLI_INTERVAL * 1000;
+ /// Static result values for ShortMonthName() function.
+ /// Short month names are also used in DateParser.
+ static const std::string MONTH_ARRAY[12];
+
/// Parse and initialize format string if it is a constant. Raise error if invalid.
static void UnixAndFromUnixPrepare(FunctionContext* context,
FunctionContext::FunctionStateScope scope);
@@ -230,11 +234,9 @@ class TimestampFunctions {
private:
- /// Static result values for DayName(), ShortDayName(), ShortMonthName() and
- /// LongMonthName functions.
+ /// Static result values for DayName(), ShortDayName() and LongMonthName() functions.
static const std::string DAY_ARRAY[7];
static const std::string DAYNAME_ARRAY[7];
- static const std::string MONTH_ARRAY[12];
static const std::string MONTHNAME_ARRAY[12];
};
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index f4049a0..f3a3554 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -32,6 +32,9 @@ add_library(Runtime
client-cache.cc
coordinator.cc
coordinator-backend-state.cc
+ datetime-parse-util.cc
+ date-parse-util.cc
+ date-value.cc
debug-options.cc
descriptors.cc
dml-exec-state.cc
@@ -81,6 +84,7 @@ ADD_BE_LSAN_TEST(mem-pool-test)
ADD_BE_LSAN_TEST(free-pool-test)
ADD_BE_LSAN_TEST(string-buffer-test)
ADD_BE_TEST(data-stream-test) # TODO: this test leaks
+ADD_BE_LSAN_TEST(date-test)
ADD_BE_LSAN_TEST(timestamp-test)
ADD_BE_LSAN_TEST(raw-value-test)
ADD_BE_LSAN_TEST(string-compare-test)
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/date-parse-util.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/date-parse-util.cc b/be/src/runtime/date-parse-util.cc
new file mode 100644
index 0000000..4217e8c
--- /dev/null
+++ b/be/src/runtime/date-parse-util.cc
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/date-parse-util.h"
+
+#include <boost/date_time/gregorian/gregorian.hpp>
+
+#include "cctz/civil_time.h"
+#include "exprs/timestamp-functions.h"
+#include "runtime/string-value.inline.h"
+#include "runtime/timestamp-value.h"
+#include "util/string-parser.h"
+
+#include "common/names.h"
+
+using boost::gregorian::date;
+using boost::posix_time::ptime;
+using boost::posix_time::time_duration;
+
+namespace impala {
+
+using datetime_parse_util::DEFAULT_DATE_CTX;
+using datetime_parse_util::DEFAULT_DATE_FMT_LEN;
+
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::DateTimeFormatToken;
+using datetime_parse_util::DateTimeParseResult;
+
+using datetime_parse_util::IsParseCtxInitialized;
+using datetime_parse_util::ParseDateTime;
+using datetime_parse_util::ParseFormatTokensByStr;
+
+using datetime_parse_util::YEAR;
+using datetime_parse_util::MONTH_IN_YEAR;
+using datetime_parse_util::MONTH_IN_YEAR_SLT;
+using datetime_parse_util::DAY_IN_MONTH;
+using datetime_parse_util::SEPARATOR;
+
+bool DateParser::Parse(const char* str, int len, const DateTimeFormatContext& dt_ctx,
+ DateValue* date) {
+ DCHECK(IsParseCtxInitialized());
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(dt_ctx.has_date_toks && !dt_ctx.has_time_toks);
+ DCHECK(date != nullptr);
+
+ DateTimeParseResult dt_result;
+ if (UNLIKELY(str == nullptr || len <= 0
+ || !ParseDateTime(str, len, dt_ctx, &dt_result))) {
+ *date = DateValue();
+ return false;
+ }
+
+ int year = dt_result.realign_year ?
+ RealignYear(dt_result, dt_ctx.century_break_ptime) :
+ dt_result.year;
+
+ *date = DateValue(year, dt_result.month, dt_result.day);
+ return date->IsValid();
+}
+
+int DateParser::RealignYear(const DateTimeParseResult& dt_result,
+ const ptime& century_break) {
+ DCHECK(dt_result.realign_year);
+ DCHECK(dt_result.year < 100);
+ DCHECK(!century_break.is_special());
+
+ const date& cb_date = century_break.date();
+ const time_duration& cb_time = century_break.time_of_day();
+ DCHECK(!cb_time.is_negative());
+
+ // Let the century start at AABB and the year parsed be YY, this gives us AAYY.
+ int year = dt_result.year + (cb_date.year() / 100) * 100;
+
+ cctz::civil_day aligned_day(year, dt_result.month, dt_result.day);
+ cctz::civil_day century_break_day(cb_date.year(), cb_date.month(), cb_date.day());
+
+ // Advance 100 years if parsed time is before the century break.
+ // For example if the century breaks at 1937 but year = 1936, the correct year would
+ // be 2036.
+ if (aligned_day < century_break_day
+ || (aligned_day == century_break_day && (cb_time.ticks() > 0))) {
+ year += 100;
+ }
+
+ return year;
+}
+
+bool DateParser::Parse(const char* str, int len, DateValue* date) {
+ DCHECK(IsParseCtxInitialized());
+ DCHECK(date != nullptr);
+ if (UNLIKELY(str == nullptr)) {
+ *date = DateValue();
+ return false;
+ }
+
+ int trimmed_len = len;
+ // Remove leading white space.
+ while (trimmed_len > 0 && isspace(*str)) {
+ ++str;
+ --trimmed_len;
+ }
+ // Strip the trailing blanks.
+ while (trimmed_len > 0 && isspace(str[trimmed_len - 1])) --trimmed_len;
+
+ if (UNLIKELY(trimmed_len <= 0)) {
+ *date = DateValue();
+ return false;
+ }
+
+ // Determine if input is a default date string. If it is, use the default date context.
+ if (LIKELY(trimmed_len == DEFAULT_DATE_FMT_LEN && str[4] == '-' && str[7] == '-')) {
+ return Parse(str, DEFAULT_DATE_FMT_LEN, DEFAULT_DATE_CTX, date);
+ }
+
+ // Generating context lazily as a fall back if default formats fail.
+ // ParseFormatTokenByStr() does not require a template format string.
+ DateTimeFormatContext lazy_ctx(str, trimmed_len);
+ if (!ParseFormatTokensByStr(&lazy_ctx, false)) {
+ *date = DateValue();
+ return false;
+ }
+ return Parse(str, trimmed_len, lazy_ctx, date);
+}
+
+int DateParser::Format(const DateTimeFormatContext& dt_ctx, const DateValue& date,
+ int len, char* buff) {
+ DCHECK(IsParseCtxInitialized());
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(dt_ctx.has_date_toks && !dt_ctx.has_time_toks);
+ DCHECK(len > dt_ctx.fmt_out_len);
+ DCHECK(buff != nullptr);
+
+ int year, month, day;
+ if (!date.ToYearMonthDay(&year, &month, &day)) return -1;
+
+ char* str = buff;
+ for (const DateTimeFormatToken& tok: dt_ctx.toks) {
+ int32_t num_val = -1;
+ const char* str_val = nullptr;
+ int str_val_len = 0;
+ switch (tok.type) {
+ case YEAR: {
+ num_val = year;
+ if (tok.len <= 3) num_val %= 100;
+ break;
+ }
+ case MONTH_IN_YEAR: num_val = month; break;
+ case MONTH_IN_YEAR_SLT: {
+ str_val = TimestampFunctions::MONTH_ARRAY[month - 1].c_str();
+ str_val_len = 3;
+ break;
+ }
+ case DAY_IN_MONTH: num_val = day; break;
+ case SEPARATOR: {
+ str_val = tok.val;
+ str_val_len = tok.len;
+ break;
+ }
+ default: DCHECK(false) << "Unknown date format token";
+ }
+ if (num_val > -1) {
+ str += sprintf(str, "%0*d", tok.len, num_val);
+ } else {
+ memcpy(str, str_val, str_val_len);
+ str += str_val_len;
+ }
+ }
+ /// Terminate the string
+ *str = '\0';
+ return str - buff;
+}
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/date-parse-util.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/date-parse-util.h b/be/src/runtime/date-parse-util.h
new file mode 100644
index 0000000..6825672
--- /dev/null
+++ b/be/src/runtime/date-parse-util.h
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "runtime/date-value.h"
+#include "runtime/datetime-parse-util.h"
+
+namespace impala {
+
+/// Used for parsing both default and custom formatted date values.
+class DateParser {
+ public:
+ /// Parse a default date string. The default timestamp format is: yyyy-MM-dd.
+ /// str -- valid pointer to the string to parse
+ /// len -- length of the string to parse (must be > 0)
+ /// date -- the date value where the results of the parsing will be placed
+ /// Returns true if the date was successfully parsed.
+ static bool Parse(const char* str, int len, DateValue* date);
+
+ /// Parse a date string. The date string must adhere to the format, otherwise it will be
+ /// rejected i.e. no missing tokens.
+ /// str -- valid pointer to the string to parse
+ /// len -- length of the string to parse (must be > 0)
+ /// dt_ctx -- date format context (must contain valid tokens)
+ /// date -- the date value where the results of the parsing will be placed
+ /// Returns true if the date was successfully parsed.
+ static bool Parse(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx, DateValue* date);
+
+ /// Format the date values using the given format context. Note that a string terminator
+ /// will be appended to the string.
+ /// dt_ctx -- date format context
+ /// date -- the date value
+ /// len -- the output buffer length (should be at least dt_ctx.fmt_exp_len + 1)
+ /// buff -- the output string buffer (must be large enough to hold value)
+ /// Return the number of characters copied in to the buffer (excluding terminator).
+ static int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx,
+ const DateValue& date, int len, char* buff);
+
+ private:
+ /// Helper function finding the correct century for 1 or 2 digit year according to
+ /// century break. The century break behavior is copied from Java SimpleDateFormat in
+ /// order to be consistent with Hive.
+ /// In SimpleDateFormat, the century for 2-digit-year breaks at current_time - 80 years.
+ /// https://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html
+ /// Returns the realigned year value for dt_result.year.
+ /// This function should be called only if dt_result.realign_year is true and
+ /// dt_result.year < 100.
+ static int RealignYear(const datetime_parse_util::DateTimeParseResult& dt_result,
+ const boost::posix_time::ptime& century_break);
+};
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/date-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/date-test.cc b/be/src/runtime/date-test.cc
new file mode 100644
index 0000000..603f7ca
--- /dev/null
+++ b/be/src/runtime/date-test.cc
@@ -0,0 +1,553 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+
+#include "common/status.h"
+#include "runtime/date-value.h"
+#include "runtime/datetime-parse-util.h"
+#include "runtime/timestamp-value.h"
+#include "testutil/gtest-util.h"
+
+#include "common/names.h"
+
+using boost::gregorian::date;
+using boost::posix_time::time_duration;
+
+namespace impala {
+
+using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::ParseFormatTokens;
+
+inline void ValidateDate(const DateValue& dv, int exp_year, int exp_month, int exp_day,
+ const string& desc) {
+ int year, month, day;
+ EXPECT_TRUE(dv.ToYearMonthDay(&year, &month, &day)) << desc;
+ EXPECT_EQ(exp_year, year);
+ EXPECT_EQ(exp_month, month);
+ EXPECT_EQ(exp_day, day);
+}
+
+inline DateValue ParseValidateDate(const char* s, int exp_year, int exp_month,
+ int exp_day) {
+ DCHECK(s != nullptr);
+ DateValue v = DateValue::Parse(s, strlen(s));
+ ValidateDate(v, exp_year, exp_month, exp_day, s);
+ return v;
+}
+
+TEST(DateTest, ParseDefault) {
+ const DateValue v1 = ParseValidateDate("2012-01-20", 2012, 1, 20);
+ const DateValue v2 = ParseValidateDate("1990-10-20", 1990, 10, 20);
+ const DateValue v3 = ParseValidateDate("1990-10-20", 1990, 10, 20);
+
+ // Test comparison operators.
+ EXPECT_NE(v1, v2);
+ EXPECT_EQ(v2, v3);
+ EXPECT_LT(v2, v1);
+ EXPECT_LE(v2, v1);
+ EXPECT_GT(v1, v2);
+ EXPECT_GE(v2, v3);
+
+ // 1-digit months and days are ok in date string.
+ ParseValidateDate("2012-1-20", 2012, 1, 20);
+ ParseValidateDate("2012-9-8", 2012, 9, 8);
+
+ // Bad formats
+ for (const char* s: {"1990-10", "1991-10-32", "1990-10-", "10:11:12 1991-10-10",
+ "2012-01-20 10:11:12", "2012-1-2 10:11:12", "02011-01-01", "999-01-01",
+ "2012-01-200", "2011-001-01"}) {
+ EXPECT_FALSE(DateValue::Parse(s, strlen(s)).IsValid()) << s;
+ }
+}
+
+// Used to represent a parsed date token. For example, it may represent a year.
+struct DateToken {
+ const char* fmt;
+ int val;
+ const char* month_name;
+
+ DateToken(const char* fmt, int val)
+ : fmt(fmt),
+ val(val),
+ month_name(nullptr) {
+ }
+
+ DateToken(const char* month_fmt, int month_val, const char* month_name)
+ : fmt(month_fmt),
+ val(month_val),
+ month_name(month_name) {
+ }
+
+ friend bool operator<(const DateToken& lhs, const DateToken& rhs) {
+ return strcmp(lhs.fmt, rhs.fmt) < 0;
+ }
+};
+
+void TestDateTokens(const vector<DateToken>& toks, int year, int month, int day,
+ const char* separator) {
+ string fmt, val;
+ for (int i = 0; i < toks.size(); ++i) {
+ fmt.append(toks[i].fmt);
+ if (separator != nullptr && i + 1 < toks.size()) fmt.push_back(*separator);
+
+ if (toks[i].month_name != nullptr) {
+ val.append(string(toks[i].month_name));
+ } else {
+ val.append(lexical_cast<string>(toks[i].val));
+ }
+ if (separator != nullptr && i + 1 < toks.size()) val.push_back(*separator);
+ }
+
+ string fmt_val = "Format: " + fmt + ", Val: " + val;
+ DateTimeFormatContext dt_ctx(fmt.c_str());
+ ASSERT_TRUE(ParseFormatTokens(&dt_ctx, false)) << fmt_val;
+ DateValue dv = DateValue::Parse(val.c_str(), val.length(), dt_ctx);
+ ValidateDate(dv, year, month, day, fmt_val);
+
+ vector<char> buff(dt_ctx.fmt_out_len + 1);
+ int actual_len = dv.Format(dt_ctx, buff.size(), buff.data());
+ EXPECT_GT(actual_len, 0) << fmt_val;
+ EXPECT_LE(actual_len, dt_ctx.fmt_len) << fmt_val;
+
+ string buff_str(buff.data());
+ EXPECT_EQ(buff_str, val) << fmt_val << " " << buff_str;
+}
+
+// This function will generate all permutations of tokens to test that the parsing and
+// formatting is correct (position of tokens should be irrelevant). Note that separators
+// are also combined with EACH token permutation to get the widest coverage on formats.
+// This forces out the parsing and format logic edge cases.
+void TestDateTokenPermutations(vector<DateToken>* toks, int year, int month, int day) {
+ sort(toks->begin(), toks->end());
+
+ const char* SEPARATORS = " ~!@%^&*_+-:;|\\,./";
+ do {
+ // Validate we can parse date raw tokens (no separators)
+ TestDateTokens(*toks, year, month, day, nullptr);
+
+ // Validate we can parse date with separators
+ for (const char* separator = SEPARATORS; *separator != 0; ++separator) {
+ TestDateTokens(*toks, year, month, day, separator);
+ }
+ } while (next_permutation(toks->begin(), toks->end()));
+}
+
+TEST(DateTest, ParseFormatCustomFormats) {
+ // Test custom formats by generating all permutations of tokens to check parsing and
+ // formatting is behaving correctly (position of tokens should be irrelevant). Note
+ // that separators are also combined with EACH token permutation to get the widest
+ // coverage on formats.
+ const int YEAR = 2013;
+ const int MONTH = 10;
+ const int DAY = 14;
+ // Test parsing/formatting with numeric date tokens
+ vector<DateToken> dt_toks{
+ DateToken("dd", DAY),
+ DateToken("MM", MONTH),
+ DateToken("yyyy", YEAR)};
+ TestDateTokenPermutations(&dt_toks, YEAR, MONTH, DAY);
+}
+
+TEST(DateTest, ParseFormatLiteralMonths) {
+ // Test literal months
+ const char* months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep",
+ "Oct", "Nov", "Dec"
+ };
+
+ // Test parsing/formatting of literal months (short)
+ const int MONTH_CNT = (sizeof(months) / sizeof(char**));
+
+ const int YEAR = 2013;
+ const int DAY = 14;
+ for (int i = 0; i < MONTH_CNT; ++i) {
+ // Test parsing/formatting with short literal months
+ vector<DateToken> dt_lm_toks{
+ DateToken("dd", DAY),
+ DateToken("MMM", i + 1, months[i]),
+ DateToken("yyyy", YEAR)};
+ TestDateTokenPermutations(&dt_lm_toks, YEAR, i + 1, DAY);
+ }
+}
+
+// Used for defining a custom date format test. The structure can be used to indicate
+// whether the format or value is expected to fail. In a happy path test, the values for
+// year, month, day will be validated against the parsed result.
+// Further validation will also be performed if the should_format flag is enabled,
+// whereby the parsed date will be translated back to a string and checked against the
+// expected value.
+struct DateTC {
+ const char* fmt;
+ const char* str;
+ bool fmt_should_fail;
+ bool str_should_fail;
+ bool should_format;
+ int expected_year;
+ int expected_month;
+ int expected_day;
+
+ DateTC(const char* fmt, const char* str, bool fmt_should_fail = true,
+ bool str_should_fail = true)
+ : fmt(fmt),
+ str(str),
+ fmt_should_fail(fmt_should_fail),
+ str_should_fail(str_should_fail),
+ should_format(true),
+ expected_year(0),
+ expected_month(0),
+ expected_day(0) {
+ }
+
+ DateTC(const char* fmt, const char* str, bool should_format,
+ int expected_year, int expected_month, int expected_day)
+ : fmt(fmt),
+ str(str),
+ fmt_should_fail(false),
+ str_should_fail(false),
+ should_format(should_format),
+ expected_year(expected_year),
+ expected_month(expected_month),
+ expected_day(expected_day) {
+ }
+
+ void Run(int id, const TimestampValue& now) const {
+ DateTimeFormatContext dt_ctx(fmt);
+ dt_ctx.SetCenturyBreak(now);
+
+ stringstream desc;
+ desc << "DateTC [" << id << "]: " << " fmt:" << fmt << " str:" << str
+ << " expected date:" << expected_year << "/" << expected_month << "/"
+ << expected_day;
+
+ bool parse_result = ParseFormatTokens(&dt_ctx, false);
+ if (fmt_should_fail) {
+ EXPECT_FALSE(parse_result) << desc.str();
+ return;
+ } else {
+ ASSERT_TRUE(parse_result) << desc.str();
+ }
+
+ DateValue cust_dv = DateValue::Parse(str, strlen(str), dt_ctx);
+ if (str_should_fail) {
+ EXPECT_FALSE(cust_dv.IsValid()) << desc.str();
+ return;
+ }
+
+ // Check the date (based on any date format tokens being present)
+ ValidateDate(cust_dv, expected_year, expected_month, expected_day, desc.str());
+
+ // Check formatted date
+ if (!should_format) return;
+
+ vector<char> buff(dt_ctx.fmt_out_len + 1);
+ int actual_len = cust_dv.Format(dt_ctx, buff.size(), buff.data());
+ EXPECT_GT(actual_len, 0) << desc.str();
+ EXPECT_LE(actual_len, dt_ctx.fmt_len) << desc.str();
+ EXPECT_EQ(string(str, strlen(str)), string(buff.data(), actual_len)) << desc.str();
+ }
+};
+
+TEST(DateTest, ParseFormatEdgeCases) {
+ const TimestampValue now(date(1980, 2, 28), time_duration(16, 14, 24));
+
+ vector<DateTC> test_cases{
+ // Test year upper/lower bound
+ DateTC("yyyy-MM-dd", "0000-01-01", true, 0, 1, 1),
+ DateTC("yyyy-MM-dd", "-001-01-01", false, true),
+ DateTC("yyyy-MM-dd", "9999-12-31", true, 9999, 12, 31),
+ DateTC("yyyyy-MM-dd", "10000-12-31", false, true),
+ // Test Feb 29 in leap years
+ DateTC("yyyy-MM-dd", "0000-02-29", true, 0, 2, 29),
+ DateTC("yyyy-MM-dd", "1904-02-29", true, 1904, 2, 29),
+ DateTC("yyyy-MM-dd", "2000-02-29", true, 2000, 2, 29),
+ // Test Feb 29 in non-leap years
+ DateTC("yyyy-MM-dd", "0001-02-29", false, true),
+ DateTC("yyyy-MM-dd", "1900-02-29", false, true),
+ DateTC("yyyy-MM-dd", "1999-02-29", false, true)};
+
+ for (int i = 0; i < test_cases.size(); ++i) {
+ test_cases[i].Run(i, now);
+ }
+}
+
+TEST(DateTest, ParseFormatSmallYear) {
+ // Fix current time to determine the behavior parsing 2-digit year format.
+ const TimestampValue now(date(1980, 2, 28), time_duration(16, 14, 24));
+
+ // Test year < 1000
+ vector<DateTC> test_cases{
+ DateTC("yyyy-MM-dd", "0999-10-31", true, 999, 10, 31),
+ DateTC("yyyy-MM-dd", "0099-10-31", true, 99, 10, 31),
+ DateTC("yyyy-MM-dd", "0009-10-31", true, 9, 10, 31),
+ // Format token yyy works when parsing years < 1000.
+ // On the other hand when yyy is used for formatting years, modulo 100 will be
+ // applied
+ DateTC("yyy-MM-dd", "999-10-31", false, 999, 10, 31),
+ DateTC("yyy-MM-dd", "099-10-31", true, 99, 10, 31),
+ DateTC("yyy-MM-dd", "009-10-31", true, 9, 10, 31),
+ // Year is aligned when yy format token is used and we have a 2-difgit year. 3-digit
+ // years are not parsed correctly.
+ DateTC("yy-MM-dd", "999-10-31", false, true),
+ DateTC("yy-MM-dd", "99-10-31", true, 1999, 10, 31),
+ DateTC("yy-MM-dd", "09-10-31", true, 1909, 10, 31),
+ // Year is aligned when y format token is used and we have a 2-digit year.
+ DateTC("y-MM-dd", "999-10-31", false, 999, 10, 31),
+ DateTC("y-MM-dd", "99-10-31", false, 1999, 10, 31),
+ DateTC("y-MM-dd", "09-10-31", false, 1909, 10, 31),
+ DateTC("y-MM-dd", "9-10-31", false, 1909, 10, 31)};
+
+ for (int i = 0; i < test_cases.size(); ++i) {
+ test_cases[i].Run(i, now);
+ }
+}
+
+TEST(DateTest, ParseFormatAlignedYear) {
+ // Fix current time to determine the behavior parsing 2-digit year format.
+ // Set it to 02/28 to test 02/29 edge cases.
+ // The corresponding century break will be 1900-02-28.
+ const TimestampValue now(date(1980, 2, 28), time_duration(16, 14, 24));
+
+ // Test year alignment for 1- and 2-digit year format.
+ vector<DateTC> test_cases{
+ // Test 2-digit year format
+ DateTC("yy-MM-dd", "17-08-31", true, 1917, 8, 31),
+ DateTC("yy-MM-dd", "99-08-31", true, 1999, 8, 31),
+ // Test 02/29 edge cases of 2-digit year format
+ DateTC("yy-MM-dd", "00-02-28", true, 2000, 2, 28),
+ // After the cutoff year is 1900, but 1900/02/29 is invalid
+ DateTC("yy-MM-dd", "00-02-29", false, true),
+ // After the cutoff year is 1900
+ DateTC("yy-MM-dd", "00-03-01", true, 1900, 3, 1),
+ DateTC("yy-MM-dd", "04-02-29", true, 1904, 2, 29),
+ DateTC("yy-MM-dd", "99-02-29", false, true),
+ // Test 1-digit year format with time to show the exact boundary
+ // Before the cutoff, year should be 2000
+ DateTC("y-MM-dd", "00-02-28", false, 2000, 2, 28),
+ // After the cutoff year is 1900, but 1900/02/29 is invalid
+ DateTC("y-MM-dd", "00-02-29", false, true),
+ // After the cutoff year is 1900.
+ DateTC("y-MM-dd", "00-03-01", false, 1900, 3, 1)};
+
+ for (int i = 0; i < test_cases.size(); ++i) {
+ test_cases[i].Run(i, now);
+ }
+
+ // Test year realignment with a different 'now' timestamp.
+ // This time the corresponding century break will be 1938-09-25.
+ const TimestampValue now2(date(2018, 9, 25), time_duration(16, 14, 24));
+
+ vector<DateTC> test_cases2{
+ // Before the cutoff, year is 2004.
+ DateTC("yy-MM-dd", "04-02-29", true, 2004, 2, 29),
+ // Still before the cutoff, year is 2038.
+ DateTC("yy-MM-dd", "38-09-25", true, 2038, 9, 25),
+ // After the cutoff, year is 1938.
+ DateTC("yy-MM-dd", "38-09-26", true, 1938, 9, 26),
+ // Test parsing again with 'y' format token.
+ DateTC("y-MM-dd", "04-02-29", false, 2004, 2, 29),
+ DateTC("y-MM-dd", "38-09-25", false, 2038, 9, 25),
+ DateTC("y-MM-dd", "38-09-26", false, 1938, 9, 26)};
+
+ for (int i = 0; i < test_cases2.size(); ++i) {
+ test_cases2[i].Run(i + test_cases.size(), now2);
+ }
+}
+
+TEST(DateTest, ParseFormatComplexFormats) {
+ const TimestampValue now(date(1980, 2, 28), time_duration(16, 14, 24));
+
+ // Test parsing/formatting of complex date formats
+ vector<DateTC> test_cases{
+ // Test case on literal short months
+ DateTC("yyyy-MMM-dd", "2013-OCT-01", false, 2013, 10, 1),
+ // Test case on literal short months
+ DateTC("yyyy-MMM-dd", "2013-oct-01", false, 2013, 10, 1),
+ // Test case on literal short months
+ DateTC("yyyy-MMM-dd", "2013-oCt-01", false, 2013, 10, 1),
+ // Test padding on numeric and literal tokens (short,
+ DateTC("MMMyyyyyydd", "Apr00201309", true, 2013, 4, 9),
+ // Test duplicate tokens
+ DateTC("yyyy MM dd ddMMMyyyy", "2013 05 12 16Apr1952", false, 1952, 4, 16),
+ // Test missing separator on short date format
+ DateTC("Myyd", "4139", true, true),
+ // Test bad year format
+ DateTC("YYYYmmdd", "20131001"),
+ // Test unknown formatting character
+ DateTC("yyyyUUdd", "2013001001"),
+ // Test that T|Z markers and time tokens are rejected
+ DateTC("yyyy-MM-ddT", "2013-11-12T"),
+ DateTC("yyyy-MM-ddZ", "2013-11-12Z"),
+ DateTC("yyyy-MM-dd HH:mm:ss", "2013-11-12 12:23:36"),
+ DateTC("HH:mm:ss", "12:23:36"),
+ // Test numeric formatting character
+ DateTC("yyyyMM1dd", "201301111"),
+ // Test out of range year
+ DateTC("yyyyyMMdd", "120130101", false, true),
+ // Test out of range month
+ DateTC("yyyyMMdd", "20131301", false, true),
+ // Test out of range month
+ DateTC("yyyyMMdd", "20130001", false, true),
+ // Test out of range day
+ DateTC("yyyyMMdd", "20130132", false, true),
+ // Test out of range day
+ DateTC("yyyyMMdd", "20130100", false, true),
+ // Test characters where numbers should be
+ DateTC("yyyyMMdd", "201301aa", false, true),
+ // Test missing year
+ DateTC("MMdd", "1201", false, true),
+ // Test missing month
+ DateTC("yyyydd", "201301", false, true),
+ DateTC("yydd", "1301", false, true),
+ // Test missing day
+ DateTC("yyyyMM", "201301", false, true),
+ DateTC("yyMM", "8512", false, true),
+ // Test missing month and day
+ DateTC("yyyy", "2013", false, true),
+ DateTC("yy", "13", false, true),
+ // Test short year token
+ DateTC("y-MM-dd", "2013-11-13", false, 2013, 11, 13),
+ DateTC("y-MM-dd", "13-11-13", false, 1913, 11, 13),
+ // Test short month token
+ DateTC("yyyy-M-dd", "2013-11-13", false, 2013, 11, 13),
+ DateTC("yyyy-M-dd", "2013-1-13", false, 2013, 1, 13),
+ // Test short day token
+ DateTC("yyyy-MM-d", "2013-11-13", false, 2013, 11, 13),
+ DateTC("yyyy-MM-d", "2013-11-3", false, 2013, 11, 3),
+ // Test short all date tokens
+ DateTC("y-M-d", "2013-11-13", false, 2013, 11, 13),
+ DateTC("y-M-d", "13-1-3", false, 1913, 1, 3)};
+
+ // Loop through custom parse/format test cases and execute each one. Each test case
+ // will be explicitly set with a pass/fail expectation related to either the format
+ // or literal value.
+ for (int i = 0; i < test_cases.size(); ++i) {
+ test_cases[i].Run(i, now);
+ }
+}
+
+// Used to test custom date output test cases i.e. date value -> string.
+struct DateFormatTC {
+ const int32_t days_since_epoch;
+ const char* fmt;
+ const char* str;
+ bool should_fail;
+
+ DateFormatTC(int32_t days_since_epoch, const char* fmt, const char* str,
+ bool should_fail = false)
+ : days_since_epoch(days_since_epoch),
+ fmt(fmt),
+ str(str),
+ should_fail(should_fail) {
+ }
+
+ void Run(int id, const TimestampValue& now) const {
+ DateTimeFormatContext dt_ctx(fmt);
+ dt_ctx.SetCenturyBreak(now);
+
+ stringstream desc;
+ desc << "DateFormatTC [" << id << "]: " << "days_since_epoch:" << days_since_epoch
+ << " fmt:" << fmt << " str:" << str;
+
+ ASSERT_TRUE(ParseFormatTokens(&dt_ctx, false)) << desc.str();
+
+ const DateValue cust_dv(days_since_epoch);
+ EXPECT_TRUE(cust_dv.IsValid()) << desc.str();
+ EXPECT_GE(dt_ctx.fmt_out_len, dt_ctx.fmt_len) << desc.str();
+
+ vector<char> buff(dt_ctx.fmt_out_len + 1);
+ int actual_len = cust_dv.Format(dt_ctx, buff.size(), buff.data());
+ EXPECT_GT(actual_len, 0) << desc.str();
+ EXPECT_LE(actual_len, dt_ctx.fmt_out_len) << desc.str();
+ EXPECT_EQ(string(buff.data(), actual_len), string(str, strlen(str))) << desc.str();
+ }
+
+};
+
+TEST(DateTest, FormatComplexFormats) {
+ const TimestampValue now(date(1980, 2, 28), time_duration(16, 14, 24));
+
+ // Test complex formatting of dates
+ vector<DateFormatTC> fmt_test_cases{
+ // Test just formatting date tokens
+ DateFormatTC(11178, "yyyy-MM-dd", "2000-08-09"),
+ // Test short form date tokens
+ DateFormatTC(11178, "yyyy-M-d", "2000-8-9"),
+ // Test short form tokens on wide dates
+ DateFormatTC(15999, "d", "21"),
+ // Test month expansion
+ DateFormatTC(11178, "MMM/MM/M", "Aug/08/8"),
+ // Test padding on single digits
+ DateFormatTC(11178, "dddddd/dd/d", "000009/09/9"),
+ // Test padding on double digits
+ DateFormatTC(15999, "dddddd/dd/dd", "000021/21/21")};
+
+ // Loop through format test cases
+ for (int i = 0; i < fmt_test_cases.size(); ++i) {
+ fmt_test_cases[i].Run(i, now);
+ }
+}
+
+TEST(DateTest, DateValueEdgeCases) {
+ // Test min supported date.
+ // MIN_DATE_DAYS_SINCE_EPOCH was calculated using the Proleptic Gregorian calendar. This
+ // is expected to be different then how Hive written Parquet files represent 0000-01-01.
+ const int32_t MIN_DATE_DAYS_SINCE_EPOCH = -719528;
+ const DateValue min_date = ParseValidateDate("0000-01-01", 0, 1, 1);
+ int32_t min_days;
+ EXPECT_TRUE(min_date.ToDaysSinceEpoch(&min_days));
+ EXPECT_EQ(MIN_DATE_DAYS_SINCE_EPOCH, min_days);
+ EXPECT_EQ("0000-01-01", min_date.ToString());
+
+ const DateValue min_date2(MIN_DATE_DAYS_SINCE_EPOCH);
+ EXPECT_TRUE(min_date2.IsValid());
+ EXPECT_EQ(min_date, min_date2);
+
+ const DateValue too_early(MIN_DATE_DAYS_SINCE_EPOCH - 1);
+ EXPECT_FALSE(too_early.IsValid());
+
+ // Test max supported date.
+ const int32_t MAX_DATE_DAYS_SINCE_EPOCH = 2932896;
+ const DateValue max_date = ParseValidateDate("9999-12-31", 9999, 12, 31);
+ int32_t max_days;
+ EXPECT_TRUE(max_date.ToDaysSinceEpoch(&max_days));
+ EXPECT_EQ(MAX_DATE_DAYS_SINCE_EPOCH, max_days);
+ EXPECT_EQ("9999-12-31", max_date.ToString());
+
+ const DateValue max_date2(MAX_DATE_DAYS_SINCE_EPOCH);
+ EXPECT_TRUE(max_date2.IsValid());
+ EXPECT_EQ(max_date, max_date2);
+
+ const DateValue too_late(MAX_DATE_DAYS_SINCE_EPOCH + 1);
+ EXPECT_FALSE(too_late.IsValid());
+
+ // Test that Feb 29 is valid in leap years.
+ for (int leap_year: {0, 1904, 1980, 1996, 2000, 2004, 2104, 9996}) {
+ EXPECT_TRUE(DateValue(leap_year, 2, 29).IsValid()) << "year:" << leap_year;
+ }
+
+ // Test that Feb 29 is invalid in non-leap years.
+ for (int non_leap_year: {1, 1900, 1981, 1999, 2001, 2100, 9999}) {
+ EXPECT_TRUE(DateValue(non_leap_year, 2, 28).IsValid()) << "year:" << non_leap_year;
+ EXPECT_FALSE(DateValue(non_leap_year, 2, 29).IsValid()) << "year:" << non_leap_year;
+ EXPECT_TRUE(DateValue(non_leap_year, 3, 1).IsValid()) << "year:" << non_leap_year;
+ }
+}
+
+}
+
+IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/date-value.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/date-value.cc b/be/src/runtime/date-value.cc
new file mode 100644
index 0000000..7223290
--- /dev/null
+++ b/be/src/runtime/date-value.cc
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include "runtime/date-value.h"
+
+#include <iomanip>
+
+#include "cctz/civil_time.h"
+#include "runtime/date-parse-util.h"
+
+#include "common/names.h"
+
+namespace impala {
+
+namespace {
+const cctz::civil_day EPOCH_DATE(1970, 1, 1);
+
+inline int32_t CalcDaysSinceEpoch(const cctz::civil_day& date) {
+ return date - EPOCH_DATE;
+}
+
+}
+
+using datetime_parse_util::DateTimeFormatContext;
+
+const int DateValue::MIN_YEAR = 0;
+const int DateValue::MAX_YEAR = 9999;
+
+const int32_t DateValue::MIN_DAYS_SINCE_EPOCH = CalcDaysSinceEpoch(
+ cctz::civil_day(MIN_YEAR, 1, 1));
+const int32_t DateValue::MAX_DAYS_SINCE_EPOCH = CalcDaysSinceEpoch(
+ cctz::civil_day(MAX_YEAR, 12, 31));
+
+DateValue::DateValue(int year, int month, int day)
+ : days_since_epoch_(INVALID_DAYS_SINCE_EPOCH) {
+ DCHECK(!IsValid());
+ // Check year range and whether year-month-day is a valid date.
+ if (LIKELY(year >= MIN_YEAR && year <= MAX_YEAR)) {
+ // Use CCTZ for validity check.
+ cctz::civil_day date(year, month, day);
+ if (LIKELY(year == date.year() && month == date.month() && day == date.day())) {
+ days_since_epoch_ = CalcDaysSinceEpoch(date);
+ DCHECK(IsValid());
+ }
+ }
+}
+
+DateValue DateValue::Parse(const char* str, int len) {
+ DateValue dv;
+ DateParser::Parse(str, len, &dv);
+ return dv;
+}
+
+DateValue DateValue::Parse(const string& str) {
+ return Parse(str.c_str(), str.size());
+}
+
+DateValue DateValue::Parse(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx) {
+ DateValue dv;
+ DateParser::Parse(str, len, dt_ctx, &dv);
+ return dv;
+}
+
+int DateValue::Format(const DateTimeFormatContext& dt_ctx, int len, char* buff) const {
+ return DateParser::Format(dt_ctx, *this, len, buff);
+}
+
+bool DateValue::ToYearMonthDay(int* year, int* month, int* day) const {
+ DCHECK(year != nullptr);
+ DCHECK(month != nullptr);
+ DCHECK(day != nullptr);
+ if (UNLIKELY(!IsValid())) return false;
+
+ const cctz::civil_day cd = EPOCH_DATE + days_since_epoch_;
+ *year = cd.year();
+ *month = cd.month();
+ *day = cd.day();
+ return true;
+}
+
+bool DateValue::ToDaysSinceEpoch(int32_t* days) const {
+ DCHECK(days != nullptr);
+ if (UNLIKELY(!IsValid())) return false;
+
+ *days = days_since_epoch_;
+ return true;
+}
+
+string DateValue::ToString() const {
+ stringstream ss;
+ int year, month, day;
+ if (ToYearMonthDay(&year, &month, &day)) {
+ ss << std::setfill('0') << setw(4) << year << "-" << setw(2) << month << "-"
+ << setw(2) << day;
+ }
+ return ss.str();
+}
+
+ostream& operator<<(ostream& os, const DateValue& date_value) {
+ return os << date_value.ToString();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/date-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/date-value.h b/be/src/runtime/date-value.h
new file mode 100644
index 0000000..17e5f01
--- /dev/null
+++ b/be/src/runtime/date-value.h
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <limits>
+#include <string>
+
+#include "common/logging.h"
+#include "common/status.h"
+
+namespace impala {
+
+namespace datetime_parse_util {
+struct DateTimeFormatContext;
+}
+
+/// Represents a DATE value.
+/// - The minimum and maximum dates are 0000-01-01 and 9999-12-31. Valid dates must fall
+/// in this range.
+/// - Internally represents DATE values as number of days since 1970.01.01.
+/// - This representation was chosen to be the same (bit-by-bit) as Parquet's date type.
+/// (https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date)
+/// - Proleptic Gregorian calendar is used to calculate the number of days since epoch,
+/// which can lead to different representation of historical dates compared to Hive.
+/// (https://en.wikipedia.org/wiki/Proleptic_Gregorian_calendar).
+/// - Supports construction from year-month-day triplets. In this case, CCTZ is used to
+/// check the validity of date.
+/// - A DateValue instance will be invalid in the following cases:
+/// - If created using the default constructor.
+/// - If constructed from out-of-range or invalid year-month-day values.
+/// - If parsed from a date string that is not valid or represents an out-of-range
+/// date.
+/// - Note, that boost::gregorian::date could not be used for representation/validation
+/// due to its limited range.
+class DateValue {
+ public:
+ /// Default constructor creates an invalid DateValue instance.
+ DateValue() : days_since_epoch_(INVALID_DAYS_SINCE_EPOCH) {
+ DCHECK(!IsValid());
+ }
+
+ DateValue(int32_t days_since_epoch) : days_since_epoch_(INVALID_DAYS_SINCE_EPOCH) {
+ DCHECK(!IsValid());
+ if (LIKELY(days_since_epoch >= MIN_DAYS_SINCE_EPOCH
+ && days_since_epoch <= MAX_DAYS_SINCE_EPOCH)) {
+ days_since_epoch_ = days_since_epoch;
+ DCHECK(IsValid());
+ }
+ }
+
+ DateValue(int year, int month, int day);
+
+ bool IsValid() const {
+ return days_since_epoch_ != INVALID_DAYS_SINCE_EPOCH;
+ }
+
+ /// If this DateValue instance is valid, return the string representation formatted as
+ /// 'yyyy-MM-dd'.
+ /// Otherwise, return empty string.
+ std::string ToString() const;
+
+ /// If this DateValue instance is valid, convert it to year-month-day and return true.
+ /// Result is placed in 'year', 'month', 'day'.
+ /// Otherwise, return false.
+ bool ToYearMonthDay(int* year, int* month, int* day) const WARN_UNUSED_RESULT;
+
+ /// If this DateValue instance is valid, convert it to the number of days since epoch
+ /// and return true. Result is placed in 'days'.
+ /// Otherwise, return false.
+ bool ToDaysSinceEpoch(int32_t* days) const WARN_UNUSED_RESULT;
+
+ /// Constructors that parse from a date string. See DateParser for details about the
+ /// date format.
+ static DateValue Parse(const char* str, int len);
+ static DateValue Parse(const std::string& str);
+ static DateValue Parse(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx);
+
+ /// Format the date using the given 'dt_ctx' format context. The result is placed in
+ /// 'buff' buffer which has the size of 'len'. The size of the buffer should be at least
+ /// dt_ctx.fmt_out_len + 1. A string terminator will be appended to the string.
+ /// Return the number of characters copied in to the buffer (excluding terminator).
+ /// If *this is invalid, nothing is placed in 'buff' and -1 is returned.
+ int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx, int len,
+ char* buff) const;
+
+ bool operator==(const DateValue& other) const {
+ return days_since_epoch_ == other.days_since_epoch_;
+ }
+ bool operator!=(const DateValue& other) const { return !(*this == other); }
+
+ bool operator<(const DateValue& other) const {
+ return days_since_epoch_ < other.days_since_epoch_;
+ }
+ bool operator>(const DateValue& other) const { return other < *this; }
+ bool operator<=(const DateValue& other) const { return !(*this > other); }
+ bool operator>=(const DateValue& other) const { return !(*this < other); }
+
+ private:
+ /// Number of days since 1970.01.01.
+ int32_t days_since_epoch_;
+
+ static const int MIN_YEAR;
+ static const int MAX_YEAR;
+
+ static const int32_t MIN_DAYS_SINCE_EPOCH;
+ static const int32_t MAX_DAYS_SINCE_EPOCH;
+ static const int32_t INVALID_DAYS_SINCE_EPOCH = std::numeric_limits<int32_t>::min();
+};
+
+std::ostream& operator<<(std::ostream& os, const DateValue& date_value);
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/datetime-parse-util.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/datetime-parse-util.cc b/be/src/runtime/datetime-parse-util.cc
new file mode 100644
index 0000000..f56642e
--- /dev/null
+++ b/be/src/runtime/datetime-parse-util.cc
@@ -0,0 +1,516 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/datetime-parse-util.h"
+
+#include <algorithm>
+
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include <boost/unordered_map.hpp>
+
+#include "cctz/civil_time.h"
+#include "exprs/timestamp-functions.h"
+#include "runtime/timestamp-value.h"
+#include "util/string-parser.h"
+
+#include "common/names.h"
+
+using boost::unordered_map;
+using boost::gregorian::date;
+using boost::gregorian::gregorian_calendar;
+using boost::posix_time::ptime;
+using boost::posix_time::time_duration;
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+void DateTimeFormatContext::SetCenturyBreak(const TimestampValue& now) {
+ const date& now_date = now.date();
+ // If the century break is at an invalid 02/29, set it to 02/28 for consistency with
+ // Hive.
+ if (now_date.month() == 2 && now_date.day() == 29 &&
+ !gregorian_calendar::is_leap_year(now_date.year() - 80)) {
+ century_break_ptime = ptime(date(now_date.year() - 80, 2, 28), now.time());
+ } else {
+ century_break_ptime = ptime(
+ date(now_date.year() - 80, now_date.month(), now_date.day()), now.time());
+ }
+}
+
+/// Used to indicate if the parsing state has been initialized.
+bool initialized = false;
+
+/// Lazily initialized pseudo-constant hashmap for mapping month names to an index.
+unordered_map<StringValue, int> REV_MONTH_INDEX;
+
+DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
+DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
+DateTimeFormatContext DEFAULT_DATE_CTX;
+DateTimeFormatContext DEFAULT_TIME_CTX;
+DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
+DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
+DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
+
+void InitParseCtx() {
+ if (initialized) return;
+ // This needs to be lazily init'd because a StringValues hash function will be invoked
+ // for each entry that's placed in the map. The hash function expects that
+ // CpuInfo::Init() has already been called.
+ REV_MONTH_INDEX = boost::unordered_map<StringValue, int>({
+ {StringValue("jan"), 1}, {StringValue("feb"), 2},
+ {StringValue("mar"), 3}, {StringValue("apr"), 4},
+ {StringValue("may"), 5}, {StringValue("jun"), 6},
+ {StringValue("jul"), 7}, {StringValue("aug"), 8},
+ {StringValue("sep"), 9}, {StringValue("oct"), 10},
+ {StringValue("nov"), 11}, {StringValue("dec"), 12}
+ });
+
+ // Setup the default date/time context yyyy-MM-dd HH:mm:ss.SSSSSSSSS
+ const char* DATE_TIME_CTX_FMT = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS";
+ const int FRACTIONAL_MAX_LEN = 9;
+ for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
+ DEFAULT_DATE_TIME_CTX[i].Reset(DATE_TIME_CTX_FMT,
+ DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
+ ParseFormatTokens(&DEFAULT_DATE_TIME_CTX[i]);
+ }
+
+ // Setup the default ISO date/time context yyyy-MM-ddTHH:mm:ss.SSSSSSSSS
+ for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
+ DEFAULT_ISO_DATE_TIME_CTX[i].Reset("yyyy-MM-ddTHH:mm:ss.SSSSSSSSS",
+ DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
+ ParseFormatTokens(&DEFAULT_ISO_DATE_TIME_CTX[i]);
+ }
+
+ // Setup the short default date/time context yyyy-MM-dd HH:mm:ss
+ DEFAULT_SHORT_DATE_TIME_CTX.Reset("yyyy-MM-dd HH:mm:ss");
+ ParseFormatTokens(&DEFAULT_SHORT_DATE_TIME_CTX);
+
+ // Setup the short default ISO date/time context yyyy-MM-ddTHH:mm:ss
+ DEFAULT_SHORT_ISO_DATE_TIME_CTX.Reset("yyyy-MM-ddTHH:mm:ss");
+ ParseFormatTokens(&DEFAULT_SHORT_ISO_DATE_TIME_CTX);
+
+ // Setup the default short date context yyyy-MM-dd
+ DEFAULT_DATE_CTX.Reset("yyyy-MM-dd");
+ ParseFormatTokens(&DEFAULT_DATE_CTX);
+
+ // Setup the default short time context HH:mm:ss
+ DEFAULT_TIME_CTX.Reset("HH:mm:ss");
+ ParseFormatTokens(&DEFAULT_TIME_CTX);
+
+ // Setup the default short time context with fractional seconds HH:mm:ss.SSSSSSSSS
+ for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
+ DEFAULT_TIME_FRAC_CTX[i].Reset(DATE_TIME_CTX_FMT + 11,
+ DEFAULT_TIME_FRAC_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
+ ParseFormatTokens(&DEFAULT_TIME_FRAC_CTX[i]);
+ }
+ // Flag that the parser is ready.
+ initialized = true;
+}
+
+bool IsParseCtxInitialized() {
+ return initialized;
+}
+
+bool IsValidTZOffset(const char* str_begin, const char* str_end) {
+ if (*str_begin == '+' || *str_begin == '-') {
+ ++str_begin;
+ switch (str_end - str_begin) {
+ case 5: // hh:mm
+ return strncmp(str_begin, "hh:mm", 5) == 0;
+ case 4: // hhmm
+ return strncmp(str_begin, "hhmm", 4) == 0;
+ case 2: // hh
+ return strncmp(str_begin, "hh", 2) == 0;
+ default:
+ break;
+ }
+ }
+ return false;
+}
+
+bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
+ DCHECK(dt_ctx != NULL);
+ DCHECK(dt_ctx->fmt != NULL);
+ DCHECK(dt_ctx->fmt_len > 0);
+ DCHECK(dt_ctx->toks.size() == 0);
+ const char* str_begin = dt_ctx->fmt;
+ const char* str_end = str_begin + dt_ctx->fmt_len;
+ const char* str = str_begin;
+ // Parse the tokens from the format string
+ while (str < str_end) {
+ if (isdigit(*str)) return false;
+
+ // If time tokens are accepted, track T|Z as separators.
+ if (*str == 'T' || *str == 'Z') {
+ if (!accept_time_toks) return false;
+ dt_ctx->toks.push_back(DateTimeFormatToken(SEPARATOR, str - str_begin, 1, str));
+ ++str;
+ continue;
+ }
+
+ // A non-alphanumerical char could be the first char of a timezone-offset token.
+ // If it is not the beginning of a time-zone offset token, track it as a separator.
+ if (!isalpha(*str)) {
+ if (dt_ctx->has_time_toks && IsValidTZOffset(str, str_end)) {
+ // TZ offset must come at the end of the format.
+ dt_ctx->toks.push_back(DateTimeFormatToken(TZ_OFFSET, str - str_begin,
+ str_end - str, str));
+ break;
+ } else {
+ dt_ctx->toks.push_back(DateTimeFormatToken(SEPARATOR, str - str_begin, 1, str));
+ ++str;
+ continue;
+ }
+ }
+
+ // Not a separator, verify that the previous token is either a separator or has
+ // length >1, i.e., it is not a variable length token.
+ if (!dt_ctx->toks.empty()) {
+ const DateTimeFormatToken& prev = dt_ctx->toks.back();
+ if (UNLIKELY(prev.type != SEPARATOR && prev.len == 1)) return false;
+ }
+ DateTimeFormatTokenType tok_type = UNKNOWN;
+ switch (*str) {
+ case 'y': tok_type = YEAR; break;
+ case 'M': tok_type = MONTH_IN_YEAR; break;
+ case 'd': tok_type = DAY_IN_MONTH; break;
+ case 'H': tok_type = HOUR_IN_DAY; break;
+ case 'm': tok_type = MINUTE_IN_HOUR; break;
+ case 's': tok_type = SECOND_IN_MINUTE; break;
+ case 'S': tok_type = FRACTION; break;
+ // Error on aA-zZ reserved characters that are not used yet.
+ default: return false;
+ }
+ dt_ctx->has_date_toks |= tok_type < HOUR_IN_DAY;
+ dt_ctx->has_time_toks |= tok_type >= HOUR_IN_DAY;
+ if (!accept_time_toks && dt_ctx->has_time_toks) return false;
+
+ // Get the token group length
+ int tok_len = 1;
+ char tok_chr = *str;
+ const char* curr_tok_chr = str + 1;
+ while (curr_tok_chr < str_end) {
+ if (*curr_tok_chr != tok_chr) break;
+ ++tok_len;
+ ++curr_tok_chr;
+ }
+ if (tok_type == MONTH_IN_YEAR) {
+ if (UNLIKELY(tok_len > 3)) return false;
+ if (tok_len == 3) tok_type = MONTH_IN_YEAR_SLT;
+ }
+ // In an output scenario, fmt_out_len is used to determine the print buffer size.
+ // If the format uses short token groups e.g. yyyy-MM-d, there must to be enough
+ // room in the buffer for wider values e.g. 2013-12-16.
+ if (tok_len == 1) ++dt_ctx->fmt_out_len;
+ DateTimeFormatToken tok(tok_type, str - str_begin, tok_len, str);
+ str += tok.len;
+ dt_ctx->toks.push_back(tok);
+ }
+ return dt_ctx->has_date_toks || dt_ctx->has_time_toks;
+}
+
+// Parse out the next digit token from the date/time string by checking for contiguous
+// digit characters and return a pointer to the end of that token.
+// str -- pointer to the string to be parsed
+// str_end -- the pointer to the end of the string to be parsed
+// Returns the pointer within the string to the end of the valid digit token.
+const char* ParseDigitToken(const char* str, const char* str_end) {
+ const char* tok_end = str;
+ while (tok_end < str_end) {
+ if (!isdigit(*tok_end)) return tok_end;
+ ++tok_end;
+ }
+ return tok_end;
+}
+
+// Parse out the next separator token from the date/time string against an expected
+// character.
+// str -- pointer to the string to be parsed
+// str_end -- the pointer to the end of the string to be parsed
+// sep -- the separator char to compare the token to
+// Returns the pointer within the string to the end of the valid separator token.
+const char* ParseSeparatorToken(const char* str, const char* str_end, const char sep) {
+ const char* tok_end = str;
+ while (tok_end < str_end) {
+ if (*tok_end != sep) return tok_end;
+ ++tok_end;
+ }
+ return tok_end;
+}
+
+bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
+ DCHECK(dt_ctx != NULL);
+ DCHECK(dt_ctx->fmt != NULL);
+ DCHECK_GT(dt_ctx->fmt_len, 0);
+ DCHECK_EQ(dt_ctx->toks.size(), 0);
+ const char* str_begin = dt_ctx->fmt;
+ const char* str_end = str_begin + dt_ctx->fmt_len;
+ const char* str = str_begin;
+ const char* tok_end;
+
+ // Parse the 4-digit year
+ tok_end = ParseDigitToken(str, str_end);
+ if (tok_end - str == 4) {
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(YEAR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Check for the date separator '-'
+ tok_end = ParseSeparatorToken(str, str_end, '-');
+ if (tok_end - str != 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Parse the 1 or 2 digit month.
+ tok_end = ParseDigitToken(str, str_end);
+ if (tok_end - str != 1 && tok_end - str != 2) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(MONTH_IN_YEAR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Check for the date separator '-'
+ tok_end = ParseSeparatorToken(str, str_end, '-');
+ if (tok_end - str != 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Parse the 1 or 2 digit day in month
+ tok_end = ParseDigitToken(str, str_end);
+ if (tok_end - str != 1 && tok_end - str != 2) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(DAY_IN_MONTH, str - str_begin, tok_end - str, str));
+ str = tok_end;
+ dt_ctx->has_date_toks = true;
+
+ // If the string ends here, we only have a date component
+ if (str == str_end) return true;
+ // If time tokens are not accepted, string should have ended here.
+ if (!accept_time_toks) return false;
+
+ // Check for the space between date and time component
+ if (*str != ' ' && *str != 'T') return false;
+ char sep = *str;
+ tok_end = ParseSeparatorToken(str, str_end, sep);
+ if (tok_end - str < 1) return false;
+ // IMPALA-6641: Multiple spaces are okay, 'T' separator must be single
+ if (sep == 'T' && tok_end - str > 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Invalid format if date-time separator is not followed by more digits
+ if (str > str_end) return false;
+ tok_end = ParseDigitToken(str, str_end);
+ }
+
+ // If time tokens are not accepted, no need to proceed.
+ if (!accept_time_toks) return false;
+
+ // Parse the 1 or 2 digit hour
+ if (tok_end - str != 1 && tok_end - str != 2) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(HOUR_IN_DAY, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Check for the time component separator ':'
+ tok_end = ParseSeparatorToken(str, str_end, ':');
+ if (tok_end - str != 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Parse the 1 or 2 digit minute
+ tok_end = ParseDigitToken(str, str_end);
+ if (tok_end - str != 1 && tok_end - str != 2) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(MINUTE_IN_HOUR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Check for the time component separator ':'
+ tok_end = ParseSeparatorToken(str, str_end, ':');
+ if (tok_end - str != 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Parse the 1 or 2 digit second
+ tok_end = ParseDigitToken(str, str_end);
+ if (tok_end - str != 1 && tok_end - str != 2) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SECOND_IN_MINUTE, str - str_begin, tok_end - str, str));
+ str = tok_end;
+ dt_ctx->has_time_toks = true;
+
+ // There is more to parse, there maybe a fractional component.
+ if (str < str_end) {
+ tok_end = ParseSeparatorToken(str, str_end, '.');
+ if (tok_end - str != 1) return false;
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(SEPARATOR, str - str_begin, tok_end - str, str));
+ str = tok_end;
+
+ // Invalid format when there is no fractional component following '.'
+ if (str > str_end) return false;
+
+ // Parse the fractional component.
+ // Like the non-lazy path, this will parse up to 9 fractional digits
+ tok_end = ParseDigitToken(str, str_end);
+ int num_digits = std::min<int>(9, tok_end - str);
+ dt_ctx->toks.push_back(
+ DateTimeFormatToken(FRACTION, str - str_begin, num_digits, str));
+ str = tok_end;
+
+ // Invalid format if there is more to parse after the fractional component
+ if (str < str_end) return false;
+ }
+ return true;
+}
+
+bool ParseDateTime(const char* str, int str_len, const DateTimeFormatContext& dt_ctx,
+ DateTimeParseResult* dt_result) {
+ DCHECK(dt_ctx.fmt_len > 0);
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(dt_result != NULL);
+ if (str_len <= 0 || str_len < dt_ctx.fmt_len || str == NULL) return false;
+ StringParser::ParseResult status;
+ // Keep track of the number of characters we need to shift token positions by.
+ // Variable-length tokens will result in values > 0;
+ int shift_len = 0;
+ for (const DateTimeFormatToken& tok: dt_ctx.toks) {
+ const char* tok_val = str + tok.pos + shift_len;
+ if (tok.type == SEPARATOR) {
+ if (UNLIKELY(*tok_val != *tok.val)) return false;
+ continue;
+ }
+ int tok_len = tok.len;
+ const char* str_end = str + str_len;
+ // In case of single-character tokens we scan ahead to the next separator.
+ if (UNLIKELY(tok_len == 1)) {
+ while ((tok_val + tok_len < str_end) && isdigit(*(tok_val + tok_len))) {
+ ++tok_len;
+ ++shift_len;
+ }
+ }
+ switch (tok.type) {
+ case YEAR: {
+ dt_result->year = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(dt_result->year < 0 || dt_result->year > 9999)) return false;
+ // Year in "Y" and "YY" format should be in the interval
+ // [current time - 80 years, current time + 20 years)
+ if (tok_len <= 2) dt_result->realign_year = true;
+ break;
+ }
+ case MONTH_IN_YEAR: {
+ dt_result->month = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(dt_result->month < 1 || dt_result->month > 12)) return false;
+ break;
+ }
+ case MONTH_IN_YEAR_SLT: {
+ char raw_buff[tok.len];
+ std::transform(tok_val, tok_val + tok.len, raw_buff, ::tolower);
+ StringValue buff(raw_buff, tok.len);
+ boost::unordered_map<StringValue, int>::const_iterator iter =
+ REV_MONTH_INDEX.find(buff);
+ if (UNLIKELY(iter == REV_MONTH_INDEX.end())) return false;
+ dt_result->month = iter->second;
+ break;
+ }
+ case DAY_IN_MONTH: {
+ dt_result->day = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ // TODO: Validate that the value of day is correct for the given month.
+ if (UNLIKELY(dt_result->day < 1 || dt_result->day > 31)) return false;
+ break;
+ }
+ case HOUR_IN_DAY: {
+ dt_result->hour = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(dt_result->hour < 0 || dt_result->hour > 23)) return false;
+ break;
+ }
+ case MINUTE_IN_HOUR: {
+ dt_result->minute = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(dt_result->minute < 0 || dt_result->minute > 59)) return false;
+ break;
+ }
+ case SECOND_IN_MINUTE: {
+ dt_result->second = StringParser::StringToInt<int>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(dt_result->second < 0 || dt_result->second > 59)) return false;
+ break;
+ }
+ case FRACTION: {
+ dt_result->fraction =
+ StringParser::StringToInt<int32_t>(tok_val, tok_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ // A user may specify a time of 04:30:22.1238, the parser will return 1238 for
+ // the fractional portion. This does not represent the intended value of
+ // 123800000, therefore the number must be scaled up.
+ for (int i = tok_len; i < 9; ++i) dt_result->fraction *= 10;
+ break;
+ }
+ case TZ_OFFSET: {
+ if (tok_val[0] != '+' && tok_val[0] != '-') return false;
+ int sign = tok_val[0] == '-' ? -1 : 1;
+ int minute = 0;
+ int hour = StringParser::StringToInt<int>(tok_val + 1, 2, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status ||
+ hour < 0 || hour > 23)) {
+ return false;
+ }
+ switch (tok_len) {
+ case 6: {
+ // +hh:mm
+ minute = StringParser::StringToInt<int>(tok_val + 4, 2, &status);
+ break;
+ }
+ case 5: {
+ // +hh:mm
+ minute = StringParser::StringToInt<int>(tok_val + 3, 2, &status);
+ break;
+ }
+ case 3: {
+ // +hh
+ break;
+ }
+ default: {
+ // Invalid timezone offset length.
+ return false;
+ }
+ }
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status ||
+ minute < 0 || minute > 59)) {
+ return false;
+ }
+ dt_result->tz_offset = time_duration(sign * hour, sign * minute, 0, 0);
+ break;
+ }
+ default: DCHECK(false) << "Unknown date/time format token";
+ }
+ }
+ return true;
+}
+
+} // namespace datetime_parse_util
+
+} // nmespace impala
http://git-wip-us.apache.org/repos/asf/impala/blob/cb493716/be/src/runtime/datetime-parse-util.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/datetime-parse-util.h b/be/src/runtime/datetime-parse-util.h
new file mode 100644
index 0000000..a21a08c
--- /dev/null
+++ b/be/src/runtime/datetime-parse-util.h
@@ -0,0 +1,246 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <vector>
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include <boost/date_time/posix_time/ptime.hpp>
+
+#include "common/logging.h"
+#include "runtime/string-value.inline.h"
+
+namespace impala {
+
+class TimestampValue;
+
+/// This namespace contains helper functions and types to parse default and custom
+/// formatted date-time values. They are used in TimestampParser and DateParser classes.
+namespace datetime_parse_util {
+
+/// Add support for dealing with custom date/time formats in Impala. The following
+/// date/time tokens are supported:
+/// y – Year
+/// M – Month
+/// d – Day
+/// H – Hour
+/// m – Minute
+/// s – second
+/// S – Fractional second
+///
+/// TimeZone offset formats (Must be at the end of format string):
+/// +/-hh:mm
+/// +/-hhmm
+/// +/-hh
+///
+///
+/// The token names and usage have been modeled after the SimpleDateFormat class used in
+/// Java, with only the above list of tokens being supported. All fields will consume
+/// variable length inputs when parsing an input string and must therefore use separators
+/// to specify the boundaries of the fields, with the exception of TimeZone values, which
+/// have to be of fixed width. Repeating tokens can be used to specify fields of exact
+/// witdh, e.g. in yy-MM both fields must be of exactly length two. When using fixed width
+/// fields values must be zero-padded and output values will be zero padded during
+/// formatting. There is one exception to this: a month field of length 3 will specify
+/// literal month names instead of zero padding, i.e., yyyy-MMM-dd will parse from and
+/// format to strings like 2013-Nov-21. When using fields of fixed width the separators
+/// can be omitted.
+///
+///
+/// Formatting character groups can appear in any order along with any separators
+/// except TimeZone offset.
+/// e.g.
+/// yyyy/MM/dd
+/// dd-MMM-yy
+/// (dd)(MM)(yyyy) HH:mm:ss
+/// yyyy-MM-dd HH:mm:ss+hh:mm
+/// ..etc..
+///
+/// The following features are not supported:
+/// Long literal months e.g. MMMM
+/// Nested strings e.g. “Year: “ yyyy “Month: “ mm “Day: “ dd
+/// Lazy formatting
+
+/// Used to indicate the type of a date/time format token group.
+enum DateTimeFormatTokenType {
+ UNKNOWN = 0,
+ SEPARATOR,
+ YEAR,
+ MONTH_IN_YEAR,
+ /// Indicates a short literal month e.g. MMM (Aug). Note that the month name is case
+ /// insensitive for an input scenario and printed in camel case for an output scenario.
+ MONTH_IN_YEAR_SLT,
+ DAY_IN_MONTH,
+ HOUR_IN_DAY,
+ MINUTE_IN_HOUR,
+ SECOND_IN_MINUTE,
+ /// Indicates fractional seconds e.g.14:52:36.2334. By default this provides nanosecond
+ /// resolution.
+ FRACTION,
+ TZ_OFFSET,
+};
+
+/// Used to store metadata about a token group within a date/time format.
+struct DateTimeFormatToken {
+ /// Indicates the type of date/time format token e.g. year
+ DateTimeFormatTokenType type;
+ /// The position of where this token group is supposed to start in the date/time string
+ /// to be parsed
+ int pos;
+ /// The length of the token group
+ int len;
+ /// A pointer to the date/time format string that is positioned at the start of this
+ /// token group
+ const char* val;
+
+ DateTimeFormatToken(DateTimeFormatTokenType type, int pos, int len, const char* val)
+ : type(type),
+ pos(pos),
+ len(len),
+ val(val) {
+ }
+};
+
+/// This structure is used to hold metadata for a date/time format. Each token group
+/// within the raw format is parsed and placed in this structure along with other high
+/// level information e.g. if the format contains date and/or time tokens. This context
+/// is used during date/time parsing.
+struct DateTimeFormatContext {
+ const char* fmt;
+ int fmt_len;
+ /// Holds the expanded length of fmt_len plus any required space when short format
+ /// tokens are used. The output buffer size is driven from this value. For example, in
+ /// an output scenario a user may provide the format yyyy-M-d, if the day and month
+ /// equates to 12, 21 then extra space is needed in the buffer to hold the values. The
+ /// short format type e.g. yyyy-M-d is valid where no zero padding is required on single
+ /// digits.
+ int fmt_out_len;
+ std::vector<DateTimeFormatToken> toks;
+ bool has_date_toks;
+ bool has_time_toks;
+ /// Current time - 80 years to determine the actual year when
+ /// parsing 1 or 2-digit year token.
+ boost::posix_time::ptime century_break_ptime;
+
+ DateTimeFormatContext() {
+ Reset(nullptr);
+ }
+
+ DateTimeFormatContext(const char* fmt) {
+ Reset(fmt);
+ }
+
+ DateTimeFormatContext(const char* fmt, int fmt_len) {
+ Reset(fmt, fmt_len);
+ }
+
+ /// Set the century break when parsing 1 or 2-digit year format.
+ /// When parsing 1 or 2-digit year, the year should be in the interval
+ /// [now - 80 years, now + 20 years), according to Hive.
+ void SetCenturyBreak(const TimestampValue& now);
+
+ void Reset(const char* fmt, int fmt_len) {
+ this->fmt = fmt;
+ this->fmt_len = fmt_len;
+ this->fmt_out_len = fmt_len;
+ this->has_date_toks = false;
+ this->has_time_toks = false;
+ this->toks.clear();
+ this->century_break_ptime = boost::posix_time::not_a_date_time;
+ }
+
+ void Reset(const char* fmt) {
+ Reset(fmt, (fmt == nullptr) ? 0 : strlen(fmt));
+ }
+};
+
+/// Stores the results of parsing a date/time string.
+struct DateTimeParseResult {
+ int year = -1;
+ int month = 0;
+ int day = 0;
+ int hour = 0;
+ int minute = 0;
+ int second = 0;
+ int32_t fraction = 0;
+ boost::posix_time::time_duration tz_offset =
+ boost::posix_time::time_duration(0, 0, 0, 0);
+ // Whether to realign the year for 2-digit year format
+ bool realign_year = false;
+};
+
+/// Constants to hold default format lengths.
+const int DEFAULT_DATE_FMT_LEN = 10;
+const int DEFAULT_TIME_FMT_LEN = 8;
+const int DEFAULT_TIME_FRAC_FMT_LEN = 18;
+const int DEFAULT_SHORT_DATE_TIME_FMT_LEN = 19;
+const int DEFAULT_DATE_TIME_FMT_LEN = 29;
+
+/// Pseudo-constant default date/time contexts. Backwards compatibility is provided on
+/// variable length fractional components by defining a format context for each expected
+/// length (0 - 9). This logic will be refactored when the parser supports lazy token
+/// groups.
+extern DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
+extern DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
+extern DateTimeFormatContext DEFAULT_DATE_CTX;
+extern DateTimeFormatContext DEFAULT_TIME_CTX;
+extern DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
+extern DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
+extern DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
+
+/// Initialize the static parser context which includes default date/time formats and
+/// lookup tables. This *must* be called before any of the Parse* related functions can
+/// be used.
+void InitParseCtx();
+
+/// Return true iff InitParseCtx() has been called.
+bool IsParseCtxInitialized();
+
+/// Parse the date/time format into tokens and place them in the context.
+/// dt_ctx -- date/time format context
+/// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
+/// rejected.
+/// Return true if the parse was successful.
+bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks = true);
+
+/// Parse the date/time string to generate the DateTimeFormatToken required by
+/// DateTimeFormatContext. Similar to ParseFormatTokens() this function will take the
+/// string and length, then heuristically determine whether the value contains date
+/// tokens, time tokens, or both. Unlike ParseFormatTokens, it does not require the
+/// template format string.
+/// dt_ctx -- date/time format context (must contain valid tokens)
+/// accept_time_toks -- if true, time tokens are accepted, otherwise time tokens are
+/// rejected.
+/// Return true if the date/time was successfully parsed.
+bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks = true);
+
+/// Parse a date/time string. The data must adhere to the format, otherwise it will be
+/// rejected i.e. no missing tokens.
+/// ParseDateTime() does only a basic validation on the parsed date/time values. The
+/// caller is responsible for implementing rigid data validation and range-check.
+/// str -- valid pointer to the string to parse
+/// len -- length of the string to parse (must be > 0)
+/// dt_ctx -- date/time format context (must contain valid tokens)
+/// dt_result -- the struct where the results of the parsing will be placed
+/// Return true if the date/time was successfully parsed.
+bool ParseDateTime(const char* str, int len, const DateTimeFormatContext& dt_ctx,
+ DateTimeParseResult* dt_result);
+
+}
+
+}