You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/09/19 18:54:10 UTC
[impala] branch master updated: IMPALA-8703: ISO:SQL:2016 datetime
patterns - Milestone 1
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new bca1b43 IMPALA-8703: ISO:SQL:2016 datetime patterns - Milestone 1
bca1b43 is described below
commit bca1b43efb72d839962f9e6999ca747fca7f17d9
Author: Gabor Kaszab <ga...@cloudera.com>
AuthorDate: Thu Jan 24 11:50:05 2019 +0100
IMPALA-8703: ISO:SQL:2016 datetime patterns - Milestone 1
This enhancement introduces FORMAT clause for CAST() operator that is
applicable for casts between string types and timestamp types. Instead
of accepting SimpleDateFormat patterns the FORMAT clause supports
datetime patterns following the ISO:SQL:2016 standard.
Note, the CAST() operator without the FORMAT clause still uses
Impala's implementation of SimpleDateFormat handling. Similarly, the
existing conversion functions such as to_timestamp(), from_timestamp()
etc. remain unchanged and use SimpleDateFormat. Contrary to how these
functions work the FORMAT clause must specify a string literal and
cannot be used with any other kind of a string expression.
Milestone 1 contains all the format tokens covered by the SQL
standard. Further milestones will add more functionality on top of
this list to cover functionality provided by other RDBMS systems.
List of tokens implemented by this change:
- YYYY, YYY, YY, Y: Year tokens
- RRRR, RR: Round year tokens
- MM: Month (1-12)
- DD: Day (1-31)
- DDD: Day of year (1-366)
- HH, HH12: Hour of day (1-12)
- HH24: Hour of day (0-23)
- MI: Minute (0-59)
- SS: Second (0-59)
- SSSSS: Second of day (0-86399)
- FF, FF1, ..., FF9: Fractional second
- AM, PM, A.M., P.M.: Meridiem indicators
- TZH: Timezone hour (-99-+99)
- TZM: Timezone minute (0-99)
- Separators: - . / , ' ; : space
- ISO8601 date indicators (T, Z)
Some notes about the matching algorithm:
- The parsing algorithm uses these tokens in a case insensitive
manner.
- The separators are interchangeable with each other. For example a
'-' separator in the format will match with a '.' character in the
input.
- The length of the separator sequences is handled flexibly meaning
that a single separator character in the format for instance would
match with a multi-separator sequence in the input.
- In a string type to timestamp conversion the timezone offset tokens
are parsed, expected to match with the input but they don't adjust
the result as the input is already expected to be in UTC format.
Usage example:
SELECT CAST('01-02-2019' AS TIMESTAMP FORMAT 'MM-DD-YYYY');
SELECT CAST('2019.10.10 13:30:40.123456 +01:30' AS TIMESTAMP
FORMAT 'YYYY-MM-DD HH24:MI:SS.FF9 TZH:TZM');
SELECT CAST(timestamp_column as STRING
FORMAT "YYYY MM HH12 YY") from some_table;
Change-Id: I19d8d097a45ae6f103b6cd1b2d81aad38dfd9e23
Reviewed-on: http://gerrit.cloudera.org:8080/13722
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/benchmarks/convert-timestamp-benchmark.cc | 14 +-
be/src/benchmarks/parse-timestamp-benchmark.cc | 132 ++--
be/src/common/init.cc | 5 +-
be/src/exec/text-converter.inline.h | 2 +-
be/src/exprs/CMakeLists.txt | 1 +
be/src/exprs/cast-format-expr.cc | 73 ++
be/src/exprs/cast-format-expr.h | 50 ++
be/src/exprs/cast-functions-ir.cc | 59 +-
be/src/exprs/date-functions-ir.cc | 4 -
be/src/exprs/expr-test.cc | 326 ++++-----
be/src/exprs/scalar-expr-evaluator.h | 1 +
be/src/exprs/scalar-expr.cc | 4 +
be/src/exprs/scalar-expr.h | 1 +
be/src/exprs/timestamp-functions-ir.cc | 49 +-
be/src/exprs/timestamp-functions.cc | 13 +-
be/src/exprs/timestamp-functions.h | 4 -
be/src/runtime/CMakeLists.txt | 5 +-
be/src/runtime/date-parse-util.cc | 114 ++--
be/src/runtime/date-parse-util.h | 33 +-
be/src/runtime/date-test.cc | 58 +-
be/src/runtime/date-value.cc | 52 +-
be/src/runtime/date-value.h | 18 +-
be/src/runtime/datetime-iso-sql-format-parser.cc | 286 ++++++++
be/src/runtime/datetime-iso-sql-format-parser.h | 79 +++
.../runtime/datetime-iso-sql-format-tokenizer.cc | 215 ++++++
be/src/runtime/datetime-iso-sql-format-tokenizer.h | 124 ++++
be/src/runtime/datetime-parse-util.h | 262 --------
be/src/runtime/datetime-parser-common.cc | 180 +++++
be/src/runtime/datetime-parser-common.h | 230 +++++++
...il.cc => datetime-simple-date-format-parser.cc} | 176 ++---
.../runtime/datetime-simple-date-format-parser.h | 181 +++++
be/src/runtime/runtime-state.cc | 12 +-
be/src/runtime/timestamp-parse-util.cc | 175 +++--
be/src/runtime/timestamp-parse-util.h | 51 +-
be/src/runtime/timestamp-test.cc | 142 ++--
be/src/runtime/timestamp-value.cc | 26 +-
be/src/runtime/timestamp-value.h | 18 +-
be/src/service/impala-server.cc | 7 +-
be/src/service/query-options.cc | 4 +
be/src/service/query-options.h | 5 +-
be/src/testutil/random-vector-generators.h | 4 +-
be/src/util/dict-test.cc | 6 +-
be/src/util/min-max-filter-test.cc | 8 +-
be/src/util/string-parser.h | 6 +-
common/thrift/Exprs.thrift | 6 +
common/thrift/ImpalaInternalService.thrift | 3 +
common/thrift/ImpalaService.thrift | 4 +
fe/src/main/cup/sql-parser.cup | 12 +-
.../java/org/apache/impala/analysis/CastExpr.java | 56 +-
.../apache/impala/analysis/AnalyzeExprsTest.java | 63 ++
.../org/apache/impala/analysis/ParserTest.java | 6 +
.../queries/QueryTest/cast_format_from_table.test | 109 +++
.../functional-query/queries/QueryTest/date.test | 288 ++++++++
tests/query_test/test_cast_with_format.py | 733 +++++++++++++++++++++
54 files changed, 3522 insertions(+), 973 deletions(-)
diff --git a/be/src/benchmarks/convert-timestamp-benchmark.cc b/be/src/benchmarks/convert-timestamp-benchmark.cc
index dd40f2f..104b162 100644
--- a/be/src/benchmarks/convert-timestamp-benchmark.cc
+++ b/be/src/benchmarks/convert-timestamp-benchmark.cc
@@ -36,7 +36,7 @@
#include "cctz/civil_time.h"
#include "exprs/timezone_db.h"
#include "exprs/timestamp-functions.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
#include "util/benchmark.h"
@@ -52,10 +52,7 @@ using std::uniform_int_distribution;
using std::thread;
using namespace impala;
-
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::InitParseCtx;
-using datetime_parse_util::ParseFormatTokens;
+using namespace datetime_parse_util;
// Benchmark tests for timestamp time-zone conversions
/*
@@ -169,7 +166,7 @@ fast path speedup: 10.2951
vector<TimestampValue> AddTestDataDateTimes(int n, const string& startstr) {
DateTimeFormatContext dt_ctx;
dt_ctx.Reset("yyyy-MMM-dd HH:mm:ss");
- ParseFormatTokens(&dt_ctx);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx);
random_device rd;
mt19937 gen(rd());
@@ -184,7 +181,7 @@ vector<TimestampValue> AddTestDataDateTimes(int n, const string& startstr) {
start += boost::gregorian::date_duration(dis_days(gen));
start += boost::posix_time::nanoseconds(dis_nanosec(gen));
string ts = to_simple_string(start);
- data.push_back(TimestampValue::Parse(ts.c_str(), ts.size(), dt_ctx));
+ data.push_back(TimestampValue::ParseSimpleDateFormat(ts.c_str(), ts.size(), dt_ctx));
}
return data;
@@ -709,7 +706,8 @@ int main(int argc, char* argv[]) {
PTR_CCTZ_LOCAL_TZ = TimezoneDatabase::FindTimezone(local_tz_name);
DCHECK(PTR_CCTZ_LOCAL_TZ != nullptr);
- InitParseCtx();
+ SimpleDateFormatTokenizer::InitCtx();
+ SimpleDateFormatParser::InitCtx();
const vector<TimestampValue> tsvalue_data =
AddTestDataDateTimes(1000, "1953-04-22 01:02:03");
diff --git a/be/src/benchmarks/parse-timestamp-benchmark.cc b/be/src/benchmarks/parse-timestamp-benchmark.cc
index d7bb34e..078192a 100644
--- a/be/src/benchmarks/parse-timestamp-benchmark.cc
+++ b/be/src/benchmarks/parse-timestamp-benchmark.cc
@@ -21,6 +21,8 @@
#include <vector>
#include <sstream>
#include <boost/date_time/posix_time/posix_time.hpp>
+#include "runtime/datetime-iso-sql-format-tokenizer.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/string-value.h"
#include "runtime/timestamp-parse-util.h"
#include "runtime/timestamp-value.h"
@@ -39,28 +41,29 @@ using boost::posix_time::to_iso_extended_string;
using boost::posix_time::to_simple_string;
using namespace impala;
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::InitParseCtx;
-using datetime_parse_util::ParseFormatTokens;
-
-// Benchmark for parsing timestamps.
-// Machine Info: Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz
-// ParseDate: Function Rate (iters/ms) Comparison
-// ----------------------------------------------------------------------
-// BoostStringDate 1.277 1X
-// BoostDate 1.229 0.962X
-// Impala 16.83 13.17X
+using namespace datetime_parse_util;
+
+// ParseDate: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+// (relative) (relative) (relative)
+// ---------------------------------------------------------------------------------------------------------
+// BoostStringDate 0.654 0.682 0.717 1X 1X 1X
+// BoostDate 0.642 0.667 0.691 0.981X 0.978X 0.964X
+// Impala 6.58 6.79 7.11 10.1X 9.96X 9.91X
//
-// ParseTimestamp: Function Rate (iters/ms) Comparison
-// ----------------------------------------------------------------------
-// BoostTime 0.9074 1X
-// Impala 15.01 16.54X
+// ParseTimestamp: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+// (relative) (relative) (relative)
+// ---------------------------------------------------------------------------------------------------------
+// BoostTime 0.48 0.491 0.51 1X 1X 1X
+// Impala 6.03 6.38 6.54 12.6X 13X 12.8X
//
-// ParseTimestampWithFormat:Function Rate (iters/ms) Comparison
-// ----------------------------------------------------------------------
-// BoostDateTime 0.4488 1X
-// ImpalaTimeStamp 37.41 83.35X
-// ImpalaTZTimeStamp 37.39 83.3X
+// ParseTimestampWithFormat: Function iters/ms 10%ile 50%ile 90%ile 10%ile 50%ile 90%ile
+// (relative) (relative) (relative)
+// ---------------------------------------------------------------------------------------------------------
+// BoostDateTime 0.24 0.25 0.26 1X 1X 1X
+// ImpalaSimpleDateFormatTimeStamp 16 16.7 17.2 66.8X 66.8X 66.3X
+// ImpalaSimpleDateFormatTZTimeStamp 16.1 16.7 17.1 67.2X 66.7X 65.8X
+// ImpalaIsoSqlFormatTimeStamp 8.12 8.42 8.71 33.8X 33.7X 33.5X
+// ImpalaIsoSqlFormatTZTimeStamp 8.12 8.56 8.85 33.8X 34.2X 34X
struct TestData {
vector<StringValue> data;
@@ -68,8 +71,10 @@ struct TestData {
vector<TimestampValue> result;
};
-DateTimeFormatContext dt_ctx;
-DateTimeFormatContext dt_ctx_tz;
+DateTimeFormatContext dt_ctx_simple_date_format;
+DateTimeFormatContext dt_ctx_tz_simple_date_format;
+DateTimeFormatContext dt_ctx_iso_sql_format;
+DateTimeFormatContext dt_ctx_tz_iso_sql_format;
void AddTestData(TestData* data, const string& input) {
data->memory.push_back(input);
@@ -127,12 +132,13 @@ void AddTestDataTZDateTimes(TestData* data, int n, const string& startstr) {
}
}
-void TestImpalaDate(int batch_size, void* d) {
+void TestImpalaSimpleDateFormat(int batch_size, void* d) {
TestData* data = reinterpret_cast<TestData*>(d);
for (int i = 0; i < batch_size; ++i) {
int n = data->data.size();
for (int j = 0; j < n; ++j) {
- data->result[j] = TimestampValue::Parse(data->data[j].ptr, data->data[j].len);
+ data->result[j] = TimestampValue::ParseSimpleDateFormat(data->data[j].ptr,
+ data->data[j].len);
}
}
}
@@ -169,24 +175,46 @@ void TestBoostTime(int batch_size, void* d) {
}
}
-void TestImpalaTimestamp(int batch_size, void* d) {
+void TestImpalaSimpleDateFormatTimestamp(int batch_size, void* d) {
+ TestData* data = reinterpret_cast<TestData*>(d);
+ for (int i = 0; i < batch_size; ++i) {
+ int n = data->data.size();
+ for (int j = 0; j < n; ++j) {
+ data->result[j] = TimestampValue::ParseSimpleDateFormat(data->data[j].ptr,
+ data->data[j].len, dt_ctx_simple_date_format);
+ }
+ }
+}
+
+void TestImpalaIsoSqlFormatTimestamp(int batch_size, void* d) {
TestData* data = reinterpret_cast<TestData*>(d);
for (int i = 0; i < batch_size; ++i) {
int n = data->data.size();
for (int j = 0; j < n; ++j) {
- data->result[j] = TimestampValue::Parse(data->data[j].ptr, data->data[j].len,
- dt_ctx);
+ data->result[j] = TimestampValue::ParseIsoSqlFormat(data->data[j].ptr,
+ data->data[j].len, dt_ctx_iso_sql_format);
}
}
}
-void TestImpalaTZTimestamp(int batch_size, void* d) {
+void TestImpalaIsoSqlFormatTZTimestamp(int batch_size, void* d) {
TestData* data = reinterpret_cast<TestData*>(d);
for (int i = 0; i < batch_size; ++i) {
int n = data->data.size();
for (int j = 0; j < n; ++j) {
- data->result[j] = TimestampValue::Parse(data->data[j].ptr, data->data[j].len,
- dt_ctx_tz);
+ data->result[j] = TimestampValue::ParseIsoSqlFormat(data->data[j].ptr,
+ data->data[j].len, dt_ctx_tz_iso_sql_format);
+ }
+ }
+}
+
+void TestImpalaSimpleDateFormatTZTimestamp(int batch_size, void* d) {
+ TestData* data = reinterpret_cast<TestData*>(d);
+ for (int i = 0; i < batch_size; ++i) {
+ int n = data->data.size();
+ for (int j = 0; j < n; ++j) {
+ data->result[j] = TimestampValue::ParseSimpleDateFormat(data->data[j].ptr,
+ data->data[j].len, dt_ctx_tz_simple_date_format);
}
}
}
@@ -206,14 +234,15 @@ int main(int argc, char **argv) {
CpuInfo::Init();
cout << Benchmark::GetMachineInfo() << endl;
- InitParseCtx();
+ SimpleDateFormatTokenizer::InitCtx();
+ SimpleDateFormatParser::InitCtx();
TestData dates, times, datetimes, tzdatetimes;
- AddTestDataDates(&dates, 1000, "1953-04-22");
- AddTestDataTimes(×, 1000, "01:02:03.45678");
- AddTestDataDateTimes(&datetimes, 1000, "1953-04-22 01:02:03");
- AddTestDataTZDateTimes(&tzdatetimes, 1000, "1990-04-22 01:10:03");
+ AddTestDataDates(&dates, 700, "1953-04-22");
+ AddTestDataTimes(×, 700, "01:02:03.45678");
+ AddTestDataDateTimes(&datetimes, 700, "1953-04-22 01:02:03");
+ AddTestDataTZDateTimes(&tzdatetimes, 700, "1990-04-22 01:10:03");
dates.result.resize(dates.data.size());
times.result.resize(times.data.size());
@@ -223,24 +252,37 @@ int main(int argc, char **argv) {
Benchmark date_suite("ParseDate");
date_suite.AddBenchmark("BoostStringDate", TestBoostStringDate, &dates);
date_suite.AddBenchmark("BoostDate", TestBoostDate, &dates);
- date_suite.AddBenchmark("Impala", TestImpalaDate, &dates);
+ date_suite.AddBenchmark("Impala", TestImpalaSimpleDateFormat, &dates);
Benchmark timestamp_suite("ParseTimestamp");
timestamp_suite.AddBenchmark("BoostTime", TestBoostTime, ×);
- timestamp_suite.AddBenchmark("Impala", TestImpalaDate, ×);
+ timestamp_suite.AddBenchmark("Impala", TestImpalaSimpleDateFormat, ×);
+
+ dt_ctx_simple_date_format.Reset("yyyy-MM-dd HH:mm:ss", 19);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx_simple_date_format);
+ dt_ctx_tz_simple_date_format.Reset("yyyy-MM-dd HH:mm:ss+hh:mm", 25);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx_tz_simple_date_format);
+
+ dt_ctx_iso_sql_format.Reset("YYYY-MM-DD HH24:MI:SS", 21);
+ datetime_parse_util::IsoSqlFormatTokenizer tokenizer(&dt_ctx_iso_sql_format,
+ datetime_parse_util::PARSE, true);
+ tokenizer.Tokenize();
- dt_ctx.Reset("yyyy-MM-dd HH:mm:ss", 19);
- ParseFormatTokens(&dt_ctx);
- dt_ctx_tz.Reset("yyyy-MM-dd HH:mm:ss+hh:mm", 25);
- ParseFormatTokens(&dt_ctx_tz);
+ dt_ctx_tz_iso_sql_format.Reset("YYYY-MM-DD HH24:MI:SSTZH:TZM", 28);
+ tokenizer.Reset(&dt_ctx_tz_iso_sql_format, datetime_parse_util::PARSE, true);
+ tokenizer.Tokenize();
Benchmark timestamp_with_format_suite("ParseTimestampWithFormat");
timestamp_with_format_suite.AddBenchmark("BoostDateTime",
TestBoostDateTime, &datetimes);
- timestamp_with_format_suite.AddBenchmark("ImpalaTimeStamp",
- TestImpalaTimestamp, &datetimes);
- timestamp_with_format_suite.AddBenchmark("ImpalaTZTimeStamp",
- TestImpalaTZTimestamp, &tzdatetimes);
+ timestamp_with_format_suite.AddBenchmark("ImpalaSimpleDateFormatTimeStamp",
+ TestImpalaSimpleDateFormatTimestamp, &datetimes);
+ timestamp_with_format_suite.AddBenchmark("ImpalaSimpleDateFormatTZTimeStamp",
+ TestImpalaSimpleDateFormatTZTimestamp, &tzdatetimes);
+ timestamp_with_format_suite.AddBenchmark("ImpalaIsoSqlFormatTimeStamp",
+ TestImpalaIsoSqlFormatTimestamp, &datetimes);
+ timestamp_with_format_suite.AddBenchmark("ImpalaIsoSqlFormatTZTimeStamp",
+ TestImpalaIsoSqlFormatTZTimestamp, &tzdatetimes);
cout << date_suite.Measure();
cout << endl;
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index feccbe0..dec323c 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -32,7 +32,7 @@
#include "rpc/authentication.h"
#include "rpc/thrift-util.h"
#include "runtime/bufferpool/buffer-pool.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/decimal-value.h"
#include "runtime/exec-env.h"
#include "runtime/hdfs-fs-cache.h"
@@ -309,7 +309,8 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
AtomicOps_x86CPUFeaturesInit();
#endif
impala::InitThreading();
- impala::datetime_parse_util::InitParseCtx();
+ impala::datetime_parse_util::SimpleDateFormatTokenizer::InitCtx();
+ impala::datetime_parse_util::SimpleDateFormatParser::InitCtx();
impala::SeedOpenSSLRNG();
ABORT_IF_ERROR(impala::InitAuth(argv[0]));
diff --git a/be/src/exec/text-converter.inline.h b/be/src/exec/text-converter.inline.h
index 0f5e6d9..07e8b9e 100644
--- a/be/src/exec/text-converter.inline.h
+++ b/be/src/exec/text-converter.inline.h
@@ -135,7 +135,7 @@ inline bool TextConverter::WriteSlot(const SlotDescriptor* slot_desc, Tuple* tup
break;
case TYPE_TIMESTAMP: {
TimestampValue* ts_slot = reinterpret_cast<TimestampValue*>(slot);
- *ts_slot = TimestampValue::Parse(data, len);
+ *ts_slot = TimestampValue::ParseSimpleDateFormat(data, len);
if (!ts_slot->HasDateOrTime()) {
parse_result = StringParser::PARSE_FAILURE;
}
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 29b15a0..33e081b 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -30,6 +30,7 @@ add_library(Exprs
anyval-util.cc
bit-byte-functions-ir.cc
case-expr.cc
+ cast-format-expr.cc
cast-functions-ir.cc
compound-predicates.cc
compound-predicates-ir.cc
diff --git a/be/src/exprs/cast-format-expr.cc b/be/src/exprs/cast-format-expr.cc
new file mode 100644
index 0000000..59a5c4e
--- /dev/null
+++ b/be/src/exprs/cast-format-expr.cc
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "cast-format-expr.h"
+
+#include "exprs/scalar-expr-evaluator.h"
+#include "exprs/timestamp-functions.h"
+#include "runtime/datetime-iso-sql-format-tokenizer.h"
+#include "runtime/runtime-state.h"
+#include "udf/udf.h"
+
+using namespace impala;
+using namespace datetime_parse_util;
+
+Status CastFormatExpr::OpenEvaluator(FunctionContext::FunctionStateScope scope,
+ RuntimeState* state, ScalarExprEvaluator* eval) const {
+ DCHECK(eval != nullptr);
+ RETURN_IF_ERROR(ScalarFnCall::OpenEvaluator(scope, state, eval));
+ if (scope != FunctionContext::FRAGMENT_LOCAL) return Status::OK();
+ DCHECK_GE(fn_ctx_idx_, 0);
+ FunctionContext* fn_ctx = eval->fn_context(fn_ctx_idx_);
+
+ dt_ctx_ = std::make_unique<DateTimeFormatContext>(format_.c_str(), format_.length());
+ bool accept_time_toks = (fn_ctx->GetReturnType().type != FunctionContext::TYPE_DATE &&
+ fn_ctx->GetArgType(0)->type != FunctionContext::TYPE_DATE);
+ IsoSqlFormatTokenizer format_tokenizer(dt_ctx_.get(), GetCastMode(fn_ctx),
+ accept_time_toks);
+ FormatTokenizationResult parse_result = format_tokenizer.Tokenize();
+ if (parse_result != FormatTokenizationResult::SUCCESS) {
+ ReportBadFormat(fn_ctx, parse_result, StringVal(format_.c_str()), true);
+ return Status(TErrorCode::INTERNAL_ERROR, fn_ctx->error_msg());
+ }
+ dt_ctx_->SetCenturyBreakAndCurrentTime(*fn_ctx->impl()->state()->now());
+ fn_ctx->SetFunctionState(scope, dt_ctx_.get());
+
+ return Status::OK();
+}
+
+void CastFormatExpr::CloseEvaluator(FunctionContext::FunctionStateScope scope,
+ RuntimeState* state, ScalarExprEvaluator* eval) const {
+ DCHECK(eval != nullptr);
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ DCHECK_GE(fn_ctx_idx_, 0);
+ FunctionContext* fn_ctx = eval->fn_context(fn_ctx_idx_);
+ fn_ctx->SetFunctionState(scope, nullptr);
+ }
+ ScalarFnCall::CloseEvaluator(scope, state, eval);
+}
+
+CastDirection CastFormatExpr::GetCastMode(const FunctionContext* ctx) const {
+ DCHECK(ctx != nullptr);
+ FunctionContext::TypeDesc type_desc = ctx->GetReturnType();
+ if (type_desc.type == FunctionContext::TYPE_STRING ||
+ type_desc.type == FunctionContext::TYPE_VARCHAR) {
+ return FORMAT;
+ }
+ return PARSE;
+}
+
diff --git a/be/src/exprs/cast-format-expr.h b/be/src/exprs/cast-format-expr.h
new file mode 100644
index 0000000..06ca022
--- /dev/null
+++ b/be/src/exprs/cast-format-expr.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/scalar-fn-call.h"
+#include "runtime/datetime-parser-common.h"
+#include <string>
+
+namespace impala {
+
+/// This is a wrapper class around ScalarFnCall to store additional information specific
+/// to the cast() operator.
+class CastFormatExpr : public ScalarFnCall {
+public:
+ /// Note, that it should be verified at the callsite that TExprNode.cast_expr is set.
+ CastFormatExpr(const TExprNode& node)
+ : ScalarFnCall(node), format_(node.cast_expr.cast_format) {}
+
+ /// Saves 'format_' into the FunctionContext object within the ScalarExprEvaluator.
+ Status OpenEvaluator(FunctionContext::FunctionStateScope scope, RuntimeState* state,
+ ScalarExprEvaluator* eval) const override WARN_UNUSED_RESULT;
+ void CloseEvaluator(FunctionContext::FunctionStateScope scope, RuntimeState* state,
+ ScalarExprEvaluator* eval) const override;
+private:
+ std::string format_;
+ /// Keeps track of the DateTimeFormatContext put into FunctionContext in
+ /// OpenEvaluator().
+ /// This is empty in case of FunctionContext::THREAD_LOCAL.
+ mutable std::unique_ptr<datetime_parse_util::DateTimeFormatContext> dt_ctx_;
+
+ datetime_parse_util::CastDirection GetCastMode(const FunctionContext* ctx) const;
+};
+
+}
+
diff --git a/be/src/exprs/cast-functions-ir.cc b/be/src/exprs/cast-functions-ir.cc
index 89734be..b56e66b 100644
--- a/be/src/exprs/cast-functions-ir.cc
+++ b/be/src/exprs/cast-functions-ir.cc
@@ -27,6 +27,7 @@
#include <gutil/strings/substitute.h>
#include "exprs/anyval-util.h"
+#include "exprs/cast-format-expr.h"
#include "exprs/decimal-functions.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
@@ -39,6 +40,7 @@
using namespace impala;
using namespace impala_udf;
+using namespace datetime_parse_util;
// The maximum number of characters need to represent a floating-point number (float or
// double) as a string. 24 = 17 (maximum significant digits) + 1 (decimal point) + 1 ('E')
@@ -166,18 +168,40 @@ StringVal CastFunctions::CastToStringVal(FunctionContext* ctx, const TinyIntVal&
}
StringVal CastFunctions::CastToStringVal(FunctionContext* ctx, const TimestampVal& val) {
+ DCHECK(ctx != nullptr);
if (val.is_null) return StringVal::null();
TimestampValue tv = TimestampValue::FromTimestampVal(val);
- StringVal sv = AnyValUtil::FromString(ctx, lexical_cast<string>(tv));
+ const DateTimeFormatContext* format_ctx =
+ reinterpret_cast<const DateTimeFormatContext*>(
+ ctx->GetFunctionState(FunctionContext::FRAGMENT_LOCAL));
+ StringVal sv;
+ if (format_ctx == nullptr) {
+ sv = AnyValUtil::FromString(ctx, tv.ToString());
+ } else {
+ string formatted_timestamp = tv.Format(*format_ctx);
+ if (formatted_timestamp.empty()) return StringVal::null();
+ sv = AnyValUtil::FromString(ctx, formatted_timestamp);
+ }
AnyValUtil::TruncateIfNecessary(ctx->GetReturnType(), &sv);
return sv;
}
StringVal CastFunctions::CastToStringVal(FunctionContext* ctx, const DateVal& val) {
+ DCHECK(ctx != nullptr);
if (val.is_null) return StringVal::null();
DateValue dv = DateValue::FromDateVal(val);
if (UNLIKELY(!dv.IsValid())) return StringVal::null();
- StringVal sv = AnyValUtil::FromString(ctx, dv.ToString());
+ const DateTimeFormatContext* format_ctx =
+ reinterpret_cast<const DateTimeFormatContext*>(
+ ctx->GetFunctionState(FunctionContext::FRAGMENT_LOCAL));
+ StringVal sv;
+ if (format_ctx == nullptr) {
+ sv = AnyValUtil::FromString(ctx, dv.ToString());
+ } else {
+ string formatted_date = dv.Format(*format_ctx);
+ if (formatted_date.empty()) return StringVal::null();
+ sv = AnyValUtil::FromString(ctx, formatted_date);
+ }
AnyValUtil::TruncateIfNecessary(ctx->GetReturnType(), &sv);
return sv;
}
@@ -281,9 +305,19 @@ CAST_TO_TIMESTAMP(BigIntVal);
TimestampVal CastFunctions::CastToTimestampVal(FunctionContext* ctx,
const StringVal& val) {
+ DCHECK(ctx != nullptr);
if (val.is_null) return TimestampVal::null();
- TimestampValue tv = TimestampValue::Parse(reinterpret_cast<char*>(val.ptr), val.len);
- // Return null if 'val' did not parse
+ const DateTimeFormatContext* format_ctx =
+ reinterpret_cast<const DateTimeFormatContext*>(
+ ctx->GetFunctionState(FunctionContext::FRAGMENT_LOCAL));
+ TimestampValue tv;
+ if (format_ctx != nullptr) {
+ tv = TimestampValue::ParseIsoSqlFormat(reinterpret_cast<const char*>(val.ptr),
+ val.len, *format_ctx);
+ } else {
+ tv = TimestampValue::ParseSimpleDateFormat(reinterpret_cast<const char*>(val.ptr),
+ val.len);
+ }
TimestampVal result;
tv.ToTimestampVal(&result);
return result;
@@ -308,13 +342,22 @@ TimestampVal CastFunctions::CastToTimestampVal(FunctionContext* ctx, const DateV
}
DateVal CastFunctions::CastToDateVal(FunctionContext* ctx, const StringVal& val) {
+ DCHECK(ctx != nullptr);
if (val.is_null) return DateVal::null();
- const char* stringVal = reinterpret_cast<const char*>(val.ptr);
- DateValue dv = DateValue::Parse(stringVal, val.len, true);
+ const char* string_val = reinterpret_cast<const char*>(val.ptr);
+ const DateTimeFormatContext* format_ctx =
+ reinterpret_cast<const DateTimeFormatContext*>(
+ ctx->GetFunctionState(FunctionContext::FRAGMENT_LOCAL));
+ DateValue dv;
+ if (format_ctx != nullptr) {
+ dv = DateValue::ParseIsoSqlFormat(string_val, val.len, *format_ctx);
+ } else {
+ dv = DateValue::ParseSimpleDateFormat(string_val, val.len, true);
+ }
if (UNLIKELY(!dv.IsValid())) {
- string invalidVal = string(stringVal, val.len);
+ string invalid_val = string(string_val, val.len);
ctx->SetError(Substitute("String to Date parse failed. Invalid string val: \"$0\"",
- invalidVal).c_str());
+ invalid_val).c_str());
return DateVal::null();
}
return dv.ToDateVal();
diff --git a/be/src/exprs/date-functions-ir.cc b/be/src/exprs/date-functions-ir.cc
index 0e235c4..1005ab4 100644
--- a/be/src/exprs/date-functions-ir.cc
+++ b/be/src/exprs/date-functions-ir.cc
@@ -24,7 +24,6 @@
#include "exprs/anyval-util.h"
#include "exprs/timestamp-functions.h"
#include "exprs/udf-builtins.h"
-#include "runtime/datetime-parse-util.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
#include "udf/udf.h"
@@ -34,9 +33,6 @@
namespace impala {
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::ParseFormatTokens;
-
IntVal DateFunctions::Year(FunctionContext* context, const DateVal& d_val) {
if (d_val.is_null) return IntVal::null();
DateValue dv = DateValue::FromDateVal(d_val);
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 2eca331..6634fc8 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -387,47 +387,47 @@ class ExprTest : public testing::TestWithParam<std::tuple<bool, bool>> {
void TestLastDayFunction() {
// Test common months (with and without time component).
TestTimestampValue("last_day('2003-01-02 04:24:04.1579')",
- TimestampValue::Parse("2003-01-31 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2003-01-31 00:00:00", 19));
TestTimestampValue("last_day('2003-02-02')",
- TimestampValue::Parse("2003-02-28 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-02-28 00:00:00"));
TestTimestampValue("last_day('2003-03-02 03:21:12.0058')",
- TimestampValue::Parse("2003-03-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-03-31 00:00:00"));
TestTimestampValue("last_day('2003-04-02')",
- TimestampValue::Parse("2003-04-30 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-04-30 00:00:00"));
TestTimestampValue("last_day('2003-05-02')",
- TimestampValue::Parse("2003-05-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-05-31 00:00:00"));
TestTimestampValue("last_day('2003-06-02')",
- TimestampValue::Parse("2003-06-30 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-06-30 00:00:00"));
TestTimestampValue("last_day('2003-07-02 00:01:01.125')",
- TimestampValue::Parse("2003-07-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-07-31 00:00:00"));
TestTimestampValue("last_day('2003-08-02')",
- TimestampValue::Parse("2003-08-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-08-31 00:00:00"));
TestTimestampValue("last_day('2003-09-02')",
- TimestampValue::Parse("2003-09-30 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-09-30 00:00:00"));
TestTimestampValue("last_day('2003-10-02')",
- TimestampValue::Parse("2003-10-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-10-31 00:00:00"));
TestTimestampValue("last_day('2003-11-02 12:30:16')",
- TimestampValue::Parse("2003-11-30 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-11-30 00:00:00"));
TestTimestampValue("last_day('2003-12-02')",
- TimestampValue::Parse("2003-12-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2003-12-31 00:00:00"));
// Test leap years and special cases.
TestTimestampValue("last_day('2004-02-13')",
- TimestampValue::Parse("2004-02-29 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2004-02-29 00:00:00"));
TestTimestampValue("last_day('2008-02-13')",
- TimestampValue::Parse("2008-02-29 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2008-02-29 00:00:00"));
TestTimestampValue("last_day('2000-02-13')",
- TimestampValue::Parse("2000-02-29 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2000-02-29 00:00:00"));
TestTimestampValue("last_day('1900-02-13')",
- TimestampValue::Parse("1900-02-28 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1900-02-28 00:00:00"));
TestTimestampValue("last_day('2100-02-13')",
- TimestampValue::Parse("2100-02-28 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2100-02-28 00:00:00"));
// Test corner cases.
TestTimestampValue("last_day('1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-31 00:00:00"));
TestTimestampValue("last_day('9999-12-31 23:59:59')",
- TimestampValue::Parse("9999-12-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 00:00:00"));
// Test invalid input.
TestIsNull("last_day('12202010')", TYPE_TIMESTAMP);
@@ -440,67 +440,68 @@ class ExprTest : public testing::TestWithParam<std::tuple<bool, bool>> {
void TestNextDayFunction() {
// Sequential test cases
TestTimestampValue("next_day('2016-05-01','Sunday')",
- TimestampValue::Parse("2016-05-08 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-08 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Monday')",
- TimestampValue::Parse("2016-05-02 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-02 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Tuesday')",
- TimestampValue::Parse("2016-05-03 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-03 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Wednesday')",
- TimestampValue::Parse("2016-05-04 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-04 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Thursday')",
- TimestampValue::Parse("2016-05-05 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-05 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Friday')",
- TimestampValue::Parse("2016-05-06 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-06 00:00:00", 19));
TestTimestampValue("next_day('2016-05-01','Saturday')",
- TimestampValue::Parse("2016-05-07 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2016-05-07 00:00:00", 19));
// Random test cases
TestTimestampValue("next_day('1910-01-18','SunDay')",
- TimestampValue::Parse("1910-01-23 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1910-01-23 00:00:00", 19));
TestTimestampValue("next_day('1916-06-05', 'SUN')",
- TimestampValue::Parse("1916-06-11 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1916-06-11 00:00:00", 19));
TestTimestampValue("next_day('1932-11-08','monday')",
- TimestampValue::Parse("1932-11-14 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1932-11-14 00:00:00", 19));
TestTimestampValue("next_day('1933-09-11','Mon')",
- TimestampValue::Parse("1933-09-18 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1933-09-18 00:00:00", 19));
TestTimestampValue("next_day('1934-03-21','TUeSday')",
- TimestampValue::Parse("1934-03-27 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1934-03-27 00:00:00", 19));
TestTimestampValue("next_day('1954-02-25','tuE')",
- TimestampValue::Parse("1954-03-02 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1954-03-02 00:00:00", 19));
TestTimestampValue("next_day('1965-04-18','WeDneSdaY')",
- TimestampValue::Parse("1965-04-21 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1965-04-21 00:00:00", 19));
TestTimestampValue("next_day('1966-08-29','wed')",
- TimestampValue::Parse("1966-08-31 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1966-08-31 00:00:00", 19));
TestTimestampValue("next_day('1968-07-23','tHurSday')",
- TimestampValue::Parse("1968-07-25 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1968-07-25 00:00:00", 19));
TestTimestampValue("next_day('1969-05-28','thu')",
- TimestampValue::Parse("1969-05-29 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1969-05-29 00:00:00", 19));
TestTimestampValue("next_day('1989-10-12','fRIDay')",
- TimestampValue::Parse("1989-10-13 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1989-10-13 00:00:00", 19));
TestTimestampValue("next_day('1973-10-02','frI')",
- TimestampValue::Parse("1973-10-05 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1973-10-05 00:00:00", 19));
TestTimestampValue("next_day('2000-02-29','saTUrDaY')",
- TimestampValue::Parse("2000-03-04 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2000-03-04 00:00:00", 19));
TestTimestampValue("next_day('2013-04-12','sat')",
- TimestampValue::Parse("2013-04-13 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2013-04-13 00:00:00", 19));
TestTimestampValue("next_day('2013-12-25','Saturday')",
- TimestampValue::Parse("2013-12-28 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2013-12-28 00:00:00", 19));
// Explicit timestamp conversion tests
TestTimestampValue("next_day(to_timestamp('12-27-2008', 'MM-dd-yyyy'), 'moN')",
- TimestampValue::Parse("2008-12-29 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2008-12-29 00:00:00", 19));
TestTimestampValue("next_day(to_timestamp('2007-20-10 11:22', 'yyyy-dd-MM HH:mm'),\
- 'TUeSdaY')", TimestampValue::Parse("2007-10-23 11:22:00", 19));
+ 'TUeSdaY')", TimestampValue::ParseSimpleDateFormat("2007-10-23 11:22:00", 19));
TestTimestampValue("next_day(to_timestamp('18-11-2070 09:12', 'dd-MM-yyyy HH:mm'),\
- 'WeDneSdaY')", TimestampValue::Parse("2070-11-19 09:12:00", 19));
+ 'WeDneSdaY')", TimestampValue::ParseSimpleDateFormat("2070-11-19 09:12:00", 19));
TestTimestampValue("next_day(to_timestamp('12-1900-05', 'dd-yyyy-MM'), 'tHurSday')",
- TimestampValue::Parse("1900-05-17 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1900-05-17 00:00:00", 19));
TestTimestampValue("next_day(to_timestamp('08-1987-21', 'MM-yyyy-dd'), 'FRIDAY')",
- TimestampValue::Parse("1987-08-28 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1987-08-28 00:00:00", 19));
TestTimestampValue("next_day(to_timestamp('02-04-2001', 'dd-MM-yyyy'), 'SAT')",
- TimestampValue::Parse("2001-04-07 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("2001-04-07 00:00:00", 19));
TestTimestampValue("next_day(to_timestamp('1970-01-31 00:00:00',\
- 'yyyy-MM-dd HH:mm:ss'), 'SunDay')", TimestampValue::Parse("1970-02-01 00:00:00", 19));
+ 'yyyy-MM-dd HH:mm:ss'), 'SunDay')",
+ TimestampValue::ParseSimpleDateFormat("1970-02-01 00:00:00", 19));
// Invalid input: unacceptable date parameter
TestIsNull("next_day('12202010','Saturday')", TYPE_TIMESTAMP);
@@ -1189,7 +1190,7 @@ class ExprTest : public testing::TestWithParam<std::tuple<bool, bool>> {
template<>
TimestampValue ExprTest::CreateTestTimestamp(const string& val) {
- return TimestampValue::Parse(val);
+ return TimestampValue::ParseSimpleDateFormat(val);
}
template<>
@@ -1277,12 +1278,12 @@ double ExprTest::ConvertValue<double>(const string& value) {
template <>
TimestampValue ExprTest::ConvertValue<TimestampValue>(const string& value) {
- return TimestampValue::Parse(value.data(), value.size());
+ return TimestampValue::ParseSimpleDateFormat(value.data(), value.size());
}
template <>
DateValue ExprTest::ConvertValue<DateValue>(const string& value) {
- return DateValue::Parse(value.data(), value.size());
+ return DateValue::ParseSimpleDateFormat(value.data(), value.size());
}
// We can't put this into TestValue() because GTest can't resolve
@@ -1316,7 +1317,8 @@ template<>
bool ExprTest::ParseString(const string& str, TimestampValue* val) {
boost::gregorian::date date;
boost::posix_time::time_duration time;
- bool success = TimestampParser::Parse(str.data(), str.length(), &date, &time);
+ bool success = TimestampParser::ParseSimpleDateFormat(str.data(), str.length(), &date,
+ &time);
val->set_date(date);
val->set_time(time);
return success;
@@ -1324,7 +1326,7 @@ bool ExprTest::ParseString(const string& str, TimestampValue* val) {
template<>
bool ExprTest::ParseString(const string& str, DateValue* val) {
- return DateParser::Parse(str.c_str(), str.length(), false, val);
+ return DateParser::ParseSimpleDateFormat(str.c_str(), str.length(), false, val);
}
Literal* ExprTest::CreateLiteral(const ColumnType& type, const string& str) {
@@ -3216,26 +3218,30 @@ TEST_P(ExprTest, CastExprs) {
// Test casting of lazy date and/or time format string to timestamp
TestTimestampValue(
- "cast('2001-1-2' as timestamp)", TimestampValue::Parse("2001-01-02 00:00:00"));
+ "cast('2001-1-2' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-02 00:00:00"));
TestTimestampValue(
- "cast('2001-01-3' as timestamp)", TimestampValue::Parse("2001-01-03 00:00:00"));
+ "cast('2001-01-3' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-03 00:00:00"));
TestTimestampValue(
- "cast('2001-1-21' as timestamp)", TimestampValue::Parse("2001-01-21 00:00:00"));
+ "cast('2001-1-21' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 00:00:00"));
TestTimestampValue("cast('2001-1-21 12:5:30' as timestamp)",
- TimestampValue::Parse("2001-01-21 12:05:30"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 12:05:30"));
TestTimestampValue("cast('2001-1-21 13:5:05' as timestamp)",
- TimestampValue::Parse("2001-01-21 13:05:05"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 13:05:05"));
TestTimestampValue("cast('2001-1-21 1:2:3' as timestamp)",
- TimestampValue::Parse("2001-01-21 01:02:03"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 01:02:03"));
TestTimestampValue("cast('2001-1-21 1:5:31.12345' as timestamp)",
- TimestampValue::Parse("2001-01-21 01:05:31.123450000"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 01:05:31.123450000"));
TestTimestampValue("cast('2001-1-21 1:5:31.12345678910111213' as timestamp)",
- TimestampValue::Parse("2001-01-21 01:05:31.123456789"));
- TestTimestampValue(
- "cast('1:05:1.12' as timestamp)", TimestampValue::Parse("01:05:01.120000000"));
- TestTimestampValue("cast('1:05:1' as timestamp)", TimestampValue::Parse("01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-21 01:05:31.123456789"));
+ TestTimestampValue("cast('1:05:1.12' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("01:05:01.120000000"));
+ TestTimestampValue("cast('1:05:1' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("01:05:01"));
TestTimestampValue("cast(' 2001-01-9 1:05:1 ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestIsNull("cast('2001-6' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast('01-1-21' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast('2001-1-21 12:5:3 AM' as timestamp)", TYPE_TIMESTAMP);
@@ -3292,29 +3298,29 @@ TEST_P(ExprTest, CastExprs) {
// IMPALA-6630: Test whitespace trimming mechanism when cast from string to timestamp
TestTimestampValue("cast(' \t\r\n 2001-01-09 01:05:01.123456789 \t\r\n' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
- TestTimestampValue("cast(' \t\r\n 2001-01-09T01:05:01.123456789 \t\r\n' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01.123456789"));
+ TestTimestampValue("cast(' \t\r\n 2001-01-09T01:05:02.123456789 \t\r\n' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:02.123456789"));
TestTimestampValue("cast(' \t\r\n 2001-01-09 01:05:01 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestTimestampValue("cast(' \t\r\n 2001-01-09T01:05:01 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestTimestampValue("cast(' \t\r\n 2001-01-09 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09"));
TestTimestampValue("cast(' \t\r\n 01:05:01 \t\r\n ' as timestamp)",
- TimestampValue::Parse("01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("01:05:01"));
TestTimestampValue("cast(' \t\r\n 01:05:01.123456789 \t\r\n ' as timestamp)",
- TimestampValue::Parse("01:05:01.123456789"));
+ TimestampValue::ParseSimpleDateFormat("01:05:01.123456789"));
TestTimestampValue("cast(' \t\r\n 2001-1-9 1:5:1 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestTimestampValue("cast(' \t\r\n 2001-1-9 1:5:1.12345678 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456780"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01.123456780"));
TestTimestampValue("cast(' \t\r\n 1:5:1 \t\r\n ' as timestamp)",
- TimestampValue::Parse("01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("01:05:01"));
TestTimestampValue("cast(' \t\r\n 1:5:1.12345678 \t\r\n ' as timestamp)",
- TimestampValue::Parse("01:05:01.123456780"));
+ TimestampValue::ParseSimpleDateFormat("01:05:01.123456780"));
TestTimestampValue("cast(' \t\r\n 2001-1-9 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09"));
// IMPALA-6995: whitespace-only strings should return NULL.
TestIsNull("cast(' ' as timestamp)", TYPE_TIMESTAMP);
@@ -3325,23 +3331,23 @@ TEST_P(ExprTest, CastExprs) {
// Test valid multi-space and 'T' separators between date and time
// components
TestTimestampValue("cast('2001-01-09 01:05:01' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
- TestTimestampValue("cast('2001-01-09T01:05:01' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
- TestTimestampValue("cast('2001-01-09 01:05:01.123456789101112' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
- TestTimestampValue("cast('2001-01-09T01:05:01.123456789101112' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
+ TestTimestampValue("cast('2001-01-09T01:05:10' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:10"));
+ TestTimestampValue("cast('2001-01-09 01:05:02.321456789101112' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:02.321456789"));
+ TestTimestampValue("cast('2001-01-09T01:05:03.123456789101112' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:03.123456789"));
TestTimestampValue("cast(' \t\r\n 2001-01-09 01:05:01 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestTimestampValue("cast(' \t\r\n 2001-01-09T01:05:01 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:01"));
TestTimestampValue(
- "cast(' \t\r\n 2001-01-09 01:05:01.12345678910 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
+ "cast(' \t\r\n 2001-01-09 01:05:04.12345678910 \t\r\n ' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-01-09 01:05:04.123456789"));
TestTimestampValue(
- "cast(' \t\r\n 2001-01-09T01:05:01.12345678910 \t\r\n ' as timestamp)",
- TimestampValue::Parse("2001-01-09 01:05:01.123456789"));
+ "cast(' \t\r\n 2001-02-09T01:05:01.12345678910 \t\r\n ' as timestamp)",
+ TimestampValue::ParseSimpleDateFormat("2001-02-09 01:05:01.123456789"));
// Test invalid variations of the 'T' separator
TestIsNull("cast('2001-01-09TTTTT01:05:01' as timestamp)", TYPE_TIMESTAMP);
@@ -3357,34 +3363,34 @@ TEST_P(ExprTest, CastExprs) {
// IMPALA-3163: Test precise conversion from Decimal to Timestamp.
TestTimestampValue("cast(cast(1457473016.1230 as decimal(17,4)) as timestamp)",
- TimestampValue::Parse("2016-03-08 21:36:56.123000000", 29));
+ TimestampValue::ParseSimpleDateFormat("2016-03-08 21:36:56.123000000", 29));
// 32 bit Decimal.
TestTimestampValue("cast(cast(123.45 as decimal(9,2)) as timestamp)",
- TimestampValue::Parse("1970-01-01 00:02:03.450000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1970-01-01 00:02:03.450000000", 29));
TestTimestampValue("cast(cast(-123.45 as decimal(9,2)) as timestamp)",
- TimestampValue::Parse("1969-12-31 23:57:56.550000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1969-12-31 23:57:56.550000000", 29));
// 64 bit Decimal.
TestTimestampValue("cast(cast(123.45 as decimal(18,2)) as timestamp)",
- TimestampValue::Parse("1970-01-01 00:02:03.450000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1970-01-01 00:02:03.450000000", 29));
TestTimestampValue("cast(cast(-123.45 as decimal(18,2)) as timestamp)",
- TimestampValue::Parse("1969-12-31 23:57:56.550000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1969-12-31 23:57:56.550000000", 29));
TestTimestampValue("cast(cast(-0.1 as decimal(18,10)) as timestamp)",
- TimestampValue::Parse("1969-12-31 23:59:59.900000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1969-12-31 23:59:59.900000000", 29));
TestTimestampValue("cast(cast(253402300799.99 as decimal(18, 2)) as timestamp)",
- TimestampValue::Parse("9999-12-31 23:59:59.990000000", 29));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.990000000", 29));
TestIsNull("cast(cast(260000000000.00 as decimal(18, 2)) as timestamp)",
TYPE_TIMESTAMP);
// 128 bit Decimal.
TestTimestampValue("cast(cast(123.45 as decimal(38,2)) as timestamp)",
- TimestampValue::Parse("1970-01-01 00:02:03.450000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1970-01-01 00:02:03.450000000", 29));
TestTimestampValue("cast(cast(-123.45 as decimal(38,2)) as timestamp)",
- TimestampValue::Parse("1969-12-31 23:57:56.550000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1969-12-31 23:57:56.550000000", 29));
TestTimestampValue("cast(cast(-0.1 as decimal(38,20)) as timestamp)",
- TimestampValue::Parse("1969-12-31 23:59:59.900000000", 29));
+ TimestampValue::ParseSimpleDateFormat("1969-12-31 23:59:59.900000000", 29));
TestTimestampValue("cast(cast(253402300799.99 as decimal(38, 2)) as timestamp)",
- TimestampValue::Parse("9999-12-31 23:59:59.990000000", 29));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.990000000", 29));
TestTimestampValue("cast(cast(253402300799.99 as decimal(38, 26)) as timestamp)",
- TimestampValue::Parse("9999-12-31 23:59:59.990000000", 29));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.990000000", 29));
TestIsNull("cast(cast(260000000000.00 as decimal(38, 2)) as timestamp)",
TYPE_TIMESTAMP);
// numeric_limits<int64_t>::max()
@@ -3417,18 +3423,18 @@ TEST_P(ExprTest, CastExprs) {
TestValue("cast(cast('2000-01-01 09:10:11.000000' as timestamp) as int)", TYPE_INT,
946717811);
TestTimestampValue("cast(946717811 as timestamp)",
- TimestampValue::Parse("2000-01-01 09:10:11", 19));
+ TimestampValue::ParseSimpleDateFormat("2000-01-01 09:10:11", 19));
// Timestamp <--> Int conversions boundary cases
TestValue("cast(cast('1400-01-01 00:00:00' as timestamp) as bigint)",
TYPE_BIGINT, -17987443200);
TestTimestampValue("cast(-17987443200 as timestamp)",
- TimestampValue::Parse("1400-01-01 00:00:00", 19));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00", 19));
TestIsNull("cast(-17987443201 as timestamp)", TYPE_TIMESTAMP);
TestValue("cast(cast('9999-12-31 23:59:59' as timestamp) as bigint)",
TYPE_BIGINT, 253402300799);
TestTimestampValue("cast(253402300799 as timestamp)",
- TimestampValue::Parse("9999-12-31 23:59:59", 19));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59", 19));
TestIsNull("cast(253402300800 as timestamp)", TYPE_TIMESTAMP);
// Timestamp <--> Float
@@ -3438,12 +3444,12 @@ TEST_P(ExprTest, CastExprs) {
TestValue("cast(cast('2000-01-01 09:10:11.720000' as timestamp) as double)",
TYPE_DOUBLE, 946717811.72);
TestTimestampValue("cast(cast(946717811.033 as double) as timestamp)",
- TimestampValue::Parse("2000-01-01 09:10:11.032999992", 29));
+ TimestampValue::ParseSimpleDateFormat("2000-01-01 09:10:11.032999992", 29));
TestValue("cast(cast('1400-01-01' as timestamp) as double)", TYPE_DOUBLE,
-17987443200);
TestIsNull("cast(cast(-17987443201.03 as double) as timestamp)", TYPE_TIMESTAMP);
TestTimestampValue("cast(253402300799 as timestamp)",
- TimestampValue::Parse("9999-12-31 23:59:59", 19));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59", 19));
TestIsNull("cast(253433923200 as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast(cast(null as bigint) as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast(cast(null as timestamp) as bigint)", TYPE_BIGINT);
@@ -3569,17 +3575,17 @@ TEST_P(ExprTest, CastDateExprs) {
DateValue(9999, 12, 31));
TestTimestampValue("cast(cast('2000-09-27' as date) as timestamp)",
- TimestampValue::Parse("2000-09-27", 10));
+ TimestampValue::ParseSimpleDateFormat("2000-09-27", 10));
TestTimestampValue("cast(date '2000-09-27' as timestamp)",
- TimestampValue::Parse("2000-09-27", 10));
+ TimestampValue::ParseSimpleDateFormat("2000-09-27", 10));
TestTimestampValue("cast(cast('9999-12-31' as date) as timestamp)",
- TimestampValue::Parse("9999-12-31", 10));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31", 10));
TestTimestampValue("cast(date '9999-12-31' as timestamp)",
- TimestampValue::Parse("9999-12-31", 10));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31", 10));
TestTimestampValue("cast(cast('1400-01-01' as date) as timestamp)",
- TimestampValue::Parse("1400-01-01", 10));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01", 10));
TestTimestampValue("cast(DATE '1400-01-01' as timestamp)",
- TimestampValue::Parse("1400-01-01", 10));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01", 10));
TestError("cast(cast('1399-12-31' as date) as timestamp)");
TestError("cast(date '1399-12-31' as timestamp)");
@@ -9782,113 +9788,113 @@ TEST_P(ExprTest, UuidTest) {
TEST_P(ExprTest, DateTruncTest) {
TestTimestampValue("date_trunc('MILLENNIUM', '2016-05-08 10:30:00')",
- TimestampValue::Parse("2001-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-01 00:00:00"));
TestTimestampValue("date_trunc('MILLENNIUM', '3000-12-31 23:59:59.999999999')",
- TimestampValue::Parse("2001-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-01 00:00:00"));
TestTimestampValue("date_trunc('MILLENNIUM', '3001-01-01 00:00:00')",
- TimestampValue::Parse("3001-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("3001-01-01 00:00:00"));
TestTimestampValue("date_trunc('CENTURY', '2016-05-08 10:30:00')",
- TimestampValue::Parse("2001-01-01 00:00:00 "));
+ TimestampValue::ParseSimpleDateFormat("2001-01-01 00:00:00 "));
TestTimestampValue("date_trunc('CENTURY', '2116-05-08 10:30:00')",
- TimestampValue::Parse("2101-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2101-01-01 00:00:00"));
TestTimestampValue("date_trunc('DECADE', '2116-05-08 10:30:00')",
- TimestampValue::Parse("2110-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2110-01-01 00:00:00"));
TestTimestampValue("date_trunc('YEAR', '2016-05-08 10:30:00')",
- TimestampValue::Parse("2016-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2016-01-01 00:00:00"));
TestTimestampValue("date_trunc('MONTH', '2016-05-08 00:00:00')",
- TimestampValue::Parse("2016-05-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2016-05-01 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2116-05-08 10:30:00')",
- TimestampValue::Parse("2116-05-04 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2116-05-04 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2017-01-01 10:37:03.455722111')",
- TimestampValue::Parse("2016-12-26 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2016-12-26 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2017-01-02 10:37:03.455722111')",
- TimestampValue::Parse("2017-01-02 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2017-01-02 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2017-01-07 10:37:03.455722111')",
- TimestampValue::Parse("2017-01-02 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2017-01-02 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2017-01-08 10:37:03.455722111')",
- TimestampValue::Parse("2017-01-02 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2017-01-02 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '2017-01-09 10:37:03.455722111')",
- TimestampValue::Parse("2017-01-09 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2017-01-09 00:00:00"));
TestTimestampValue("date_trunc('DAY', '1416-05-08 10:37:03.455722111')",
- TimestampValue::Parse("1416-05-08 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 00:00:00"));
TestTimestampValue("date_trunc('HOUR', '1416-05-08 10:30:03.455722111')",
- TimestampValue::Parse("1416-05-08 10:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 10:00:00"));
TestTimestampValue("date_trunc('HOUR', '1416-05-08 23:30:03.455722111')",
- TimestampValue::Parse("1416-05-08 23:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 23:00:00"));
TestTimestampValue("date_trunc('MINUTE', '1416-05-08 10:37:03.455722111')",
- TimestampValue::Parse("1416-05-08 10:37:00"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 10:37:00"));
TestTimestampValue("date_trunc('SECOND', '1416-05-08 10:37:03.455722111')",
- TimestampValue::Parse("1416-05-08 10:37:03"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 10:37:03"));
TestTimestampValue("date_trunc('MILLISECONDS', '1416-05-08 10:37:03.455722111')",
- TimestampValue::Parse("1416-05-08 10:37:03.455000000"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 10:37:03.455000000"));
TestTimestampValue("date_trunc('MICROSECONDS', '1416-05-08 10:37:03.455722111')",
- TimestampValue::Parse("1416-05-08 10:37:03.455722000"));
+ TimestampValue::ParseSimpleDateFormat("1416-05-08 10:37:03.455722000"));
// Test corner cases.
TestTimestampValue("date_trunc('MILLENNIUM', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9001-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9001-01-01 00:00:00"));
TestTimestampValue("date_trunc('CENTURY', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9901-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9901-01-01 00:00:00"));
TestTimestampValue("date_trunc('DECADE', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9990-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9990-01-01 00:00:00"));
TestTimestampValue("date_trunc('YEAR', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-01-01 00:00:00"));
TestTimestampValue("date_trunc('MONTH', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-01 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-27 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-27 00:00:00"));
TestTimestampValue("date_trunc('DAY', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 00:00:00"));
TestTimestampValue("date_trunc('HOUR', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 23:00:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:00:00"));
TestTimestampValue("date_trunc('MINUTE', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 23:59:00"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:00"));
TestTimestampValue("date_trunc('SECOND', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 23:59:59"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59"));
TestTimestampValue("date_trunc('MILLISECONDS', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 23:59:59.999"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.999"));
TestTimestampValue("date_trunc('MICROSECONDS', '9999-12-31 23:59:59.999999999')",
- TimestampValue::Parse("9999-12-31 23:59:59.999999"));
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.999999"));
TestTimestampValue("date_trunc('DECADE', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('YEAR', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('MONTH', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('DAY', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('HOUR', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('MINUTE', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('SECOND', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('MILLISECONDS', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
TestTimestampValue("date_trunc('MICROSECONDS', '1400-01-01 00:00:00')",
- TimestampValue::Parse("1400-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00"));
// Test lower limit for century
TestIsNull("date_trunc('CENTURY', '1400-01-01 00:00:00')", TYPE_TIMESTAMP);
TestIsNull("date_trunc('CENTURY', '1400-12-31 23:59:59.999999999')", TYPE_TIMESTAMP);
TestTimestampValue("date_trunc('CENTURY', '1401-01-01 00:00:00')",
- TimestampValue::Parse("1401-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1401-01-01 00:00:00"));
// Test lower limit for millennium
TestIsNull("date_trunc('MILLENNIUM', '1400-01-01 00:00:00')", TYPE_TIMESTAMP);
TestIsNull("date_trunc('MILLENNIUM', '2000-12-31 23:59:59.999999999')", TYPE_TIMESTAMP);
TestTimestampValue("date_trunc('MILLENNIUM', '2001-01-01 00:00:00')",
- TimestampValue::Parse("2001-01-01 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("2001-01-01 00:00:00"));
// Test lower limit for week
TestIsNull("date_trunc('WEEK', '1400-01-01 00:00:00')", TYPE_TIMESTAMP);
TestIsNull("date_trunc('WEEK', '1400-01-05 23:59:59.999999999')", TYPE_TIMESTAMP);
TestTimestampValue("date_trunc('WEEK', '1400-01-06 00:00:00')",
- TimestampValue::Parse("1400-01-06 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-06 00:00:00"));
TestTimestampValue("date_trunc('WEEK', '1400-01-07 23:59:59.999999999')",
- TimestampValue::Parse("1400-01-06 00:00:00"));
+ TimestampValue::ParseSimpleDateFormat("1400-01-06 00:00:00"));
// Test invalid input.
TestIsNull("date_trunc('HOUR', '12202010')", TYPE_TIMESTAMP);
diff --git a/be/src/exprs/scalar-expr-evaluator.h b/be/src/exprs/scalar-expr-evaluator.h
index e241104..cee3ad9 100644
--- a/be/src/exprs/scalar-expr-evaluator.h
+++ b/be/src/exprs/scalar-expr-evaluator.h
@@ -200,6 +200,7 @@ class ScalarExprEvaluator {
protected:
/// Users of fn_context();
friend class CaseExpr;
+ friend class CastFormatExpr;
friend class HiveUdfCall;
friend class ScalarFnCall;
diff --git a/be/src/exprs/scalar-expr.cc b/be/src/exprs/scalar-expr.cc
index ba5ccc8..54b4d76 100644
--- a/be/src/exprs/scalar-expr.cc
+++ b/be/src/exprs/scalar-expr.cc
@@ -25,6 +25,7 @@
#include "common/object-pool.h"
#include "common/status.h"
#include "exprs/case-expr.h"
+#include "exprs/cast-format-expr.h"
#include "exprs/compound-predicates.h"
#include "exprs/conditional-functions.h"
#include "exprs/hive-udf-call.h"
@@ -184,6 +185,9 @@ Status ScalarExpr::CreateNode(
*expr = pool->Add(new CoalesceExpr(texpr_node));
} else if (texpr_node.fn.binary_type == TFunctionBinaryType::JAVA) {
*expr = pool->Add(new HiveUdfCall(texpr_node));
+ } else if (texpr_node.__isset.cast_expr &&
+ !texpr_node.cast_expr.cast_format.empty()) {
+ *expr = pool->Add(new CastFormatExpr(texpr_node));
} else {
*expr = pool->Add(new ScalarFnCall(texpr_node));
}
diff --git a/be/src/exprs/scalar-expr.h b/be/src/exprs/scalar-expr.h
index 64b56aa..0da79d0 100644
--- a/be/src/exprs/scalar-expr.h
+++ b/be/src/exprs/scalar-expr.h
@@ -210,6 +210,7 @@ class ScalarExpr : public Expr {
friend class AggFnEvaluator;
friend class AndPredicate;
friend class CaseExpr;
+ friend class CastFormatExpr;
friend class CoalesceExpr;
friend class ConditionalFunctions;
friend class CompoundPredicate;
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 8426b0e..3d27340 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -24,7 +24,7 @@
#include <gutil/strings/substitute.h>
#include "exprs/anyval-util.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-value.inline.h"
#include "runtime/timestamp-value.h"
@@ -56,8 +56,7 @@ typedef boost::posix_time::seconds Seconds;
namespace impala {
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::ParseFormatTokens;
+using namespace datetime_parse_util;
StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
const TimestampValue& tv, const StringVal& fmt) {
@@ -65,17 +64,15 @@ StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!ParseFormatTokens(dt_ctx)){
- TimestampFunctions::ReportBadFormat(context, fmt, false);
+ if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx)){
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
return StringVal::null();
}
}
- int buff_len = dt_ctx->fmt_out_len + 1;
- StringVal result(context, buff_len);
- if (UNLIKELY(result.is_null)) return StringVal::null();
- result.len = tv.Format(*dt_ctx, buff_len, reinterpret_cast<char*>(result.ptr));
- if (result.len <= 0) return StringVal::null();
+ string formatted_timestamp = tv.Format(*dt_ctx);
+ if (formatted_timestamp.empty()) return StringVal::null();
+ StringVal result = AnyValUtil::FromString(context, formatted_timestamp);
return result;
}
@@ -92,7 +89,7 @@ template <class TIME>
StringVal TimestampFunctions::FromUnix(FunctionContext* context, const TIME& intp,
const StringVal& fmt) {
if (fmt.is_null || fmt.len == 0) {
- TimestampFunctions::ReportBadFormat(context, fmt, false);
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
return StringVal::null();
}
if (intp.is_null) return StringVal::null();
@@ -160,7 +157,7 @@ TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
const StringVal& date, const StringVal& fmt) {
if (fmt.is_null || fmt.len == 0) {
- TimestampFunctions::ReportBadFormat(context, fmt, false);
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
return TimestampVal::null();
}
if (date.is_null || date.len == 0) return TimestampVal::null();
@@ -168,12 +165,12 @@ TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!ParseFormatTokens(dt_ctx)) {
- ReportBadFormat(context, fmt, false);
+ if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx)) {
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
return TimestampVal::null();
}
}
- const TimestampValue& tv = TimestampValue::Parse(
+ const TimestampValue& tv = TimestampValue::ParseSimpleDateFormat(
reinterpret_cast<const char*>(date.ptr), date.len, *dt_ctx);
TimestampVal tv_val;
tv.ToTimestampVal(&tv_val);
@@ -191,29 +188,13 @@ StringVal TimestampFunctions::FromTimestamp(FunctionContext* context,
BigIntVal TimestampFunctions::UnixFromString(FunctionContext* context,
const StringVal& sv) {
if (sv.is_null) return BigIntVal::null();
- const TimestampValue& tv = TimestampValue::Parse(
+ const TimestampValue& tv = TimestampValue::ParseSimpleDateFormat(
reinterpret_cast<const char *>(sv.ptr), sv.len);
time_t result;
return (tv.ToUnixTime(context->impl()->state()->local_time_zone(), &result)) ?
BigIntVal(result) : BigIntVal::null();
}
-void TimestampFunctions::ReportBadFormat(FunctionContext* context,
- const StringVal& format, bool is_error) {
- stringstream ss;
- const StringValue& fmt = StringValue::FromStringVal(format);
- if (format.is_null || format.len == 0) {
- ss << "Bad date/time conversion format: format string is NULL or has 0 length";
- } else {
- ss << "Bad date/time conversion format: " << fmt.DebugString();
- }
- if (is_error) {
- context->SetError(ss.str().c_str());
- } else {
- context->AddWarning(ss.str().c_str());
- }
-}
-
IntVal TimestampFunctions::Year(FunctionContext* context, const TimestampVal& ts_val) {
if (ts_val.is_null) return IntVal::null();
const TimestampValue& ts_value = TimestampValue::FromTimestampVal(ts_val);
@@ -345,10 +326,6 @@ StringVal TimestampFunctions::ToDate(FunctionContext* context,
return result;
}
-inline bool IsLeapYear(int year) {
- return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
-}
-
inline unsigned short GetLastDayOfMonth(int month, int year) {
switch (month) {
case 1: return 31;
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
index 97bf1df..231c853 100644
--- a/be/src/exprs/timestamp-functions.cc
+++ b/be/src/exprs/timestamp-functions.cc
@@ -21,7 +21,7 @@
#include "exprs/anyval-util.h"
#include "exprs/timezone_db.h"
#include "gutil/strings/substitute.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
@@ -32,8 +32,7 @@
namespace impala {
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::ParseFormatTokens;
+using namespace datetime_parse_util;
const string TimestampFunctions::DAY_ARRAY[7] = {"Sun", "Mon", "Tue", "Wed", "Thu",
"Fri", "Sat"};
@@ -130,14 +129,14 @@ void TimestampFunctions::UnixAndFromUnixPrepare(
StringVal fmt_val = *reinterpret_cast<StringVal*>(context->GetConstantArg(1));
const StringValue& fmt_ref = StringValue::FromStringVal(fmt_val);
if (fmt_val.is_null || fmt_ref.len == 0) {
- TimestampFunctions::ReportBadFormat(context, fmt_val, true);
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt_val, true);
return;
}
dt_ctx = new DateTimeFormatContext(fmt_ref.ptr, fmt_ref.len);
- bool parse_result = ParseFormatTokens(dt_ctx);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(dt_ctx);
if (!parse_result) {
delete dt_ctx;
- TimestampFunctions::ReportBadFormat(context, fmt_val, true);
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt_val, true);
return;
}
} else {
@@ -146,7 +145,7 @@ void TimestampFunctions::UnixAndFromUnixPrepare(
// This is much cheaper vs alloc/dealloc'ing a context for each evaluation.
dt_ctx = new DateTimeFormatContext();
}
- dt_ctx->SetCenturyBreak(*context->impl()->state()->now());
+ dt_ctx->SetCenturyBreakAndCurrentTime(*context->impl()->state()->now());
context->SetFunctionState(scope, dt_ctx);
}
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index 3beadf1..e1d3773 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -239,10 +239,6 @@ class TimestampFunctions {
/// TODO: eventually return format converted from Java to Boost.
static StringValue* CheckFormat(StringValue* format);
- /// Issue a warning for a bad format string.
- static void ReportBadFormat(FunctionContext* context,
- const StringVal& format, bool is_error);
-
private:
/// Static result values for DayName() function.
static const std::string DAY_ARRAY[7];
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index cf7a3bd..eda4af9 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -35,7 +35,10 @@ add_library(Runtime
coordinator.cc
coordinator-backend-state.cc
coordinator-backend-resource-state.cc
- datetime-parse-util.cc
+ datetime-iso-sql-format-parser.cc
+ datetime-iso-sql-format-tokenizer.cc
+ datetime-parser-common.cc
+ datetime-simple-date-format-parser.cc
date-parse-util.cc
date-value.cc
debug-options.cc
diff --git a/be/src/runtime/date-parse-util.cc b/be/src/runtime/date-parse-util.cc
index 1ba0360..008c1b9 100644
--- a/be/src/runtime/date-parse-util.cc
+++ b/be/src/runtime/date-parse-util.cc
@@ -21,6 +21,8 @@
#include "cctz/civil_time.h"
#include "exprs/timestamp-functions.h"
+#include "runtime/datetime-iso-sql-format-parser.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-value.h"
#include "util/string-parser.h"
@@ -33,27 +35,10 @@ using boost::posix_time::time_duration;
namespace impala {
-using datetime_parse_util::DEFAULT_DATE_CTX;
-using datetime_parse_util::DEFAULT_DATE_FMT_LEN;
+using namespace datetime_parse_util;
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::DateTimeFormatToken;
-using datetime_parse_util::DateTimeParseResult;
-
-using datetime_parse_util::IsParseCtxInitialized;
-using datetime_parse_util::ParseDateTime;
-using datetime_parse_util::ParseFormatTokensByStr;
-using datetime_parse_util::ParseDefaultFormatTokensByStr;
-
-using datetime_parse_util::YEAR;
-using datetime_parse_util::MONTH_IN_YEAR;
-using datetime_parse_util::MONTH_IN_YEAR_SLT;
-using datetime_parse_util::DAY_IN_MONTH;
-using datetime_parse_util::SEPARATOR;
-
-bool DateParser::Parse(const char* str, int len, const DateTimeFormatContext& dt_ctx,
- DateValue* date) {
- DCHECK(IsParseCtxInitialized());
+bool DateParser::ParseSimpleDateFormat(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx, DateValue* date) {
DCHECK(dt_ctx.toks.size() > 0);
// 'dt_ctx' must have date tokens. Time tokens are accepted but ignored.
DCHECK(dt_ctx.has_date_toks);
@@ -61,7 +46,7 @@ bool DateParser::Parse(const char* str, int len, const DateTimeFormatContext& dt
DateTimeParseResult dt_result;
if (UNLIKELY(str == nullptr || len <= 0
- || !ParseDateTime(str, len, dt_ctx, &dt_result))) {
+ || !SimpleDateFormatParser::ParseDateTime(str, len, dt_ctx, &dt_result))) {
*date = DateValue();
return false;
}
@@ -101,13 +86,10 @@ int DateParser::RealignYear(const DateTimeParseResult& dt_result,
return year;
}
-bool DateParser::Parse(const char* str, int len, bool accept_time_toks, DateValue* date) {
- DCHECK(IsParseCtxInitialized());
+bool DateParser::ParseSimpleDateFormat(const char* str, int len, bool accept_time_toks,
+ DateValue* date) {
DCHECK(date != nullptr);
- if (UNLIKELY(str == nullptr)) {
- *date = DateValue();
- return false;
- }
+ if (UNLIKELY(str == nullptr)) return IndicateDateParseFailure(date);
int trimmed_len = len;
// Remove leading white space.
@@ -117,48 +99,59 @@ bool DateParser::Parse(const char* str, int len, bool accept_time_toks, DateValu
}
// Strip the trailing blanks.
while (trimmed_len > 0 && isspace(str[trimmed_len - 1])) --trimmed_len;
+ if (UNLIKELY(trimmed_len <= 0)) return IndicateDateParseFailure(date);
- if (UNLIKELY(trimmed_len <= 0)) {
- *date = DateValue();
- return false;
- }
-
- // Determine the default formatting context that's required for parsing.
- const DateTimeFormatContext* dt_ctx = ParseDefaultFormatTokensByStr(str, trimmed_len,
- accept_time_toks, false);
-
- if (dt_ctx != nullptr) return Parse(str, trimmed_len, *dt_ctx, date);
+ const DateTimeFormatContext* dt_ctx =
+ SimpleDateFormatTokenizer::GetDefaultFormatContext(str, trimmed_len,
+ accept_time_toks, false);
+ if (dt_ctx != nullptr) return ParseSimpleDateFormat(str, trimmed_len, *dt_ctx, date);
// Generating context lazily as a fall back if default formats fail.
// ParseFormatTokenByStr() does not require a template format string.
DateTimeFormatContext lazy_ctx(str, trimmed_len);
- if (!ParseFormatTokensByStr(&lazy_ctx, accept_time_toks, false)) {
- *date = DateValue();
- return false;
+ if (!SimpleDateFormatTokenizer::TokenizeByStr(&lazy_ctx, accept_time_toks, false)) {
+ return IndicateDateParseFailure(date);
}
- return Parse(str, trimmed_len, lazy_ctx, date);
+ return ParseSimpleDateFormat(str, trimmed_len, lazy_ctx, date);
}
-int DateParser::Format(const DateTimeFormatContext& dt_ctx, const DateValue& date,
- int len, char* buff) {
- DCHECK(IsParseCtxInitialized());
+bool DateParser::ParseIsoSqlFormat(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx, DateValue* date) {
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(date != nullptr);
+ DCHECK(dt_ctx.has_date_toks);
+
+ if (UNLIKELY(str == nullptr || len <= 0)) return IndicateDateParseFailure(date);
+
+ DateTimeParseResult dt_result;
+ bool result = IsoSqlFormatParser::ParseDateTime(str, len, dt_ctx, &dt_result);
+ if (!result) return IndicateDateParseFailure(date);
+
+ *date = DateValue(dt_result.year, dt_result.month, dt_result.day);
+ return date->IsValid();
+}
+
+string DateParser::Format(const DateTimeFormatContext& dt_ctx, const DateValue& date) {
DCHECK(dt_ctx.toks.size() > 0);
DCHECK(dt_ctx.has_date_toks && !dt_ctx.has_time_toks);
- DCHECK(len > dt_ctx.fmt_out_len);
- DCHECK(buff != nullptr);
int year, month, day;
- if (!date.ToYearMonthDay(&year, &month, &day)) return -1;
+ if (!date.ToYearMonthDay(&year, &month, &day)) return "";
- char* str = buff;
+ string result;
+ result.reserve(dt_ctx.fmt_out_len);
for (const DateTimeFormatToken& tok: dt_ctx.toks) {
int32_t num_val = -1;
const char* str_val = nullptr;
int str_val_len = 0;
switch (tok.type) {
- case YEAR: {
+ case YEAR:
+ case ROUND_YEAR: {
num_val = year;
- if (tok.len <= 3) num_val %= 100;
+ if (tok.len < 4) {
+ int adjust_factor = std::pow(10, tok.len);
+ num_val %= adjust_factor;
+ }
break;
}
case MONTH_IN_YEAR: num_val = month; break;
@@ -168,6 +161,10 @@ int DateParser::Format(const DateTimeFormatContext& dt_ctx, const DateValue& dat
break;
}
case DAY_IN_MONTH: num_val = day; break;
+ case DAY_IN_YEAR: {
+ num_val = GetDayInYear(year, month, day);
+ break;
+ }
case SEPARATOR: {
str_val = tok.val;
str_val_len = tok.len;
@@ -176,15 +173,20 @@ int DateParser::Format(const DateTimeFormatContext& dt_ctx, const DateValue& dat
default: DCHECK(false) << "Unknown date format token";
}
if (num_val > -1) {
- str += sprintf(str, "%0*d", tok.len, num_val);
+ string tmp_str = std::to_string(num_val);
+ if (tmp_str.length() < tok.len) tmp_str.insert(0, tok.len - tmp_str.length(), '0');
+ result.append(tmp_str);
} else {
- memcpy(str, str_val, str_val_len);
- str += str_val_len;
+ DCHECK(str_val != nullptr && str_val_len > 0);
+ result.append(str_val, str_val_len);
}
}
- /// Terminate the string
- *str = '\0';
- return str - buff;
+ return result;
+}
+
+bool DateParser::IndicateDateParseFailure(DateValue* date) {
+ *date = DateValue();
+ return false;
}
}
diff --git a/be/src/runtime/date-parse-util.h b/be/src/runtime/date-parse-util.h
index e9a3997..c8b8765 100644
--- a/be/src/runtime/date-parse-util.h
+++ b/be/src/runtime/date-parse-util.h
@@ -19,7 +19,7 @@
#pragma once
#include "runtime/date-value.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-parser-common.h"
namespace impala {
@@ -33,8 +33,8 @@ class DateParser {
/// rejected.
/// date -- the date value where the results of the parsing will be placed
/// Returns true if the date was successfully parsed.
- static bool Parse(const char* str, int len, bool accept_time_toks, DateValue* date)
- WARN_UNUSED_RESULT;
+ static bool ParseSimpleDateFormat(const char* str, int len, bool accept_time_toks,
+ DateValue* date) WARN_UNUSED_RESULT;
/// Parse a date string. The date string must adhere to the format, otherwise it will be
/// rejected i.e. no missing tokens.
@@ -44,19 +44,26 @@ class DateParser {
/// and ignored)
/// date -- the date value where the results of the parsing will be placed
/// Returns true if the date was successfully parsed.
- static bool Parse(const char* str, int len,
+ static bool ParseSimpleDateFormat(const char* str, int len,
const datetime_parse_util::DateTimeFormatContext& dt_ctx, DateValue* date)
WARN_UNUSED_RESULT;
- /// Format the date values using the given format context. Note that a string terminator
- /// will be appended to the string.
+ /// Parse a date string using ISO:SQL:2016 datetime patterns. The date string must
+ /// adhere to the format, otherwise it will be rejected i.e. no missing tokens.
+ /// str -- valid pointer to the string to parse
+ /// len -- length of the string to parse (must be > 0)
+ /// dt_ctx -- format context (must contain valid date tokens)
+ /// date -- the date value where the results of the parsing will be placed
+ /// Returns true if the date was successfully parsed.
+ static bool ParseIsoSqlFormat(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx, DateValue* date)
+ WARN_UNUSED_RESULT;
+
+ /// Format the date values using the given format context.
/// dt_ctx -- date format context
/// date -- the date value
- /// len -- the output buffer length (should be at least dt_ctx.fmt_exp_len + 1)
- /// buff -- the output string buffer (must be large enough to hold value)
- /// Return the number of characters copied in to the buffer (excluding terminator).
- static int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx,
- const DateValue& date, int len, char* buff);
+ static string Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx,
+ const DateValue& date);
private:
/// Helper function finding the correct century for 1 or 2 digit year according to
@@ -69,6 +76,10 @@ class DateParser {
/// dt_result.year < 100.
static int RealignYear(const datetime_parse_util::DateTimeParseResult& dt_result,
const boost::posix_time::ptime& century_break);
+
+ /// Helper for parse functions to produce return value and set output parameter to an
+ /// invalid DateValue when parsing fails.
+ static bool IndicateDateParseFailure(DateValue* date);
};
}
diff --git a/be/src/runtime/date-test.cc b/be/src/runtime/date-test.cc
index fefd418..eb37f62 100644
--- a/be/src/runtime/date-test.cc
+++ b/be/src/runtime/date-test.cc
@@ -20,8 +20,8 @@
#include "cctz/civil_time.h"
#include "common/status.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/date-value.h"
-#include "runtime/datetime-parse-util.h"
#include "runtime/raw-value.inline.h"
#include "runtime/timestamp-value.h"
#include "testutil/gtest-util.h"
@@ -33,8 +33,7 @@ using boost::posix_time::time_duration;
namespace impala {
-using datetime_parse_util::DateTimeFormatContext;
-using datetime_parse_util::ParseFormatTokens;
+using namespace datetime_parse_util;
inline void ValidateDate(const DateValue& dv, int exp_year, int exp_month, int exp_day,
const string& desc) {
@@ -48,7 +47,7 @@ inline void ValidateDate(const DateValue& dv, int exp_year, int exp_month, int e
inline DateValue ParseValidateDate(const char* s, bool accept_time_toks, int exp_year,
int exp_month, int exp_day) {
DCHECK(s != nullptr);
- DateValue v = DateValue::Parse(s, strlen(s), accept_time_toks);
+ DateValue v = DateValue::ParseSimpleDateFormat(s, strlen(s), accept_time_toks);
ValidateDate(v, exp_year, exp_month, exp_day, s);
return v;
}
@@ -98,20 +97,20 @@ TEST(DateTest, ParseDefault) {
// Bad formats: invalid date component.
for (const char* s: {"1990-10", "1991-10-32", "1990-10-", "10:11:12 1991-10-10",
"02011-01-01", "999-01-01", "2012-01-200", "2011-001-01"}) {
- EXPECT_FALSE(DateValue::Parse(s, strlen(s), false).IsValid()) << s;
+ EXPECT_FALSE(DateValue::ParseSimpleDateFormat(s, strlen(s), false).IsValid()) << s;
}
// Bad formats: valid date and time components but time component is rejected.
for (const char* s: {"2012-01-20 10:11:12", "2012-1-2 10:11:12"}) {
- EXPECT_FALSE(DateValue::Parse(s, strlen(s), false).IsValid()) << s;
+ EXPECT_FALSE(DateValue::ParseSimpleDateFormat(s, strlen(s), false).IsValid()) << s;
}
// Bad formats: valid date component, invalid time component.
for (const char* s: {"2012-01-20 10:11:", "2012-1-2 10::12", "2012-01-20 :11:12",
"2012-01-20 24:11:12", "2012-01-20 23:60:12"}) {
- EXPECT_FALSE(DateValue::Parse(s, strlen(s), true).IsValid()) << s;
+ EXPECT_FALSE(DateValue::ParseSimpleDateFormat(s, strlen(s), true).IsValid()) << s;
}
// Bad formats: missing date component, valid time component.
for (const char* s: {"10:11:12", "1:11:12", "10:1:12", "10:1:2.999"}) {
- EXPECT_FALSE(DateValue::Parse(s, strlen(s), true).IsValid()) << s;
+ EXPECT_FALSE(DateValue::ParseSimpleDateFormat(s, strlen(s), true).IsValid()) << s;
}
}
@@ -155,17 +154,14 @@ void TestDateTokens(const vector<DateToken>& toks, int year, int month, int day,
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(ParseFormatTokens(&dt_ctx, false)) << fmt_val;
- DateValue dv = DateValue::Parse(val.c_str(), val.length(), dt_ctx);
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false)) << fmt_val;
+ DateValue dv = DateValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateDate(dv, year, month, day, fmt_val);
- vector<char> buff(dt_ctx.fmt_out_len + 1);
- int actual_len = dv.Format(dt_ctx, buff.size(), buff.data());
- EXPECT_GT(actual_len, 0) << fmt_val;
- EXPECT_LE(actual_len, dt_ctx.fmt_len) << fmt_val;
-
- string buff_str(buff.data());
- EXPECT_EQ(buff_str, val) << fmt_val << " " << buff_str;
+ string buff = dv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << fmt_val;
+ EXPECT_LE(buff.length(), dt_ctx.fmt_len) << fmt_val;
+ EXPECT_EQ(buff, val) << fmt_val << " " << buff;
}
// This function will generate all permutations of tokens to test that the parsing and
@@ -266,14 +262,14 @@ struct DateTC {
void Run(int id, const TimestampValue& now) const {
DateTimeFormatContext dt_ctx(fmt);
- dt_ctx.SetCenturyBreak(now);
+ dt_ctx.SetCenturyBreakAndCurrentTime(now);
stringstream desc;
desc << "DateTC [" << id << "]: " << " fmt:" << fmt << " str:" << str
<< " expected date:" << expected_year << "/" << expected_month << "/"
<< expected_day;
- bool parse_result = ParseFormatTokens(&dt_ctx, false);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false);
if (fmt_should_fail) {
EXPECT_FALSE(parse_result) << desc.str();
return;
@@ -281,7 +277,7 @@ struct DateTC {
ASSERT_TRUE(parse_result) << desc.str();
}
- DateValue cust_dv = DateValue::Parse(str, strlen(str), dt_ctx);
+ DateValue cust_dv = DateValue::ParseSimpleDateFormat(str, strlen(str), dt_ctx);
if (str_should_fail) {
EXPECT_FALSE(cust_dv.IsValid()) << desc.str();
return;
@@ -293,11 +289,10 @@ struct DateTC {
// Check formatted date
if (!should_format) return;
- vector<char> buff(dt_ctx.fmt_out_len + 1);
- int actual_len = cust_dv.Format(dt_ctx, buff.size(), buff.data());
- EXPECT_GT(actual_len, 0) << desc.str();
- EXPECT_LE(actual_len, dt_ctx.fmt_len) << desc.str();
- EXPECT_EQ(string(str, strlen(str)), string(buff.data(), actual_len)) << desc.str();
+ string buff = cust_dv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << desc.str();
+ EXPECT_LE(buff.length(), dt_ctx.fmt_len) << desc.str();
+ EXPECT_EQ(string(str, strlen(str)), buff) << desc.str();
}
};
@@ -496,23 +491,22 @@ struct DateFormatTC {
void Run(int id, const TimestampValue& now) const {
DateTimeFormatContext dt_ctx(fmt);
- dt_ctx.SetCenturyBreak(now);
+ dt_ctx.SetCenturyBreakAndCurrentTime(now);
stringstream desc;
desc << "DateFormatTC [" << id << "]: " << "days_since_epoch:" << days_since_epoch
<< " fmt:" << fmt << " str:" << str;
- ASSERT_TRUE(ParseFormatTokens(&dt_ctx, false)) << desc.str();
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false)) << desc.str();
const DateValue cust_dv(days_since_epoch);
EXPECT_TRUE(cust_dv.IsValid()) << desc.str();
EXPECT_GE(dt_ctx.fmt_out_len, dt_ctx.fmt_len) << desc.str();
- vector<char> buff(dt_ctx.fmt_out_len + 1);
- int actual_len = cust_dv.Format(dt_ctx, buff.size(), buff.data());
- EXPECT_GT(actual_len, 0) << desc.str();
- EXPECT_LE(actual_len, dt_ctx.fmt_out_len) << desc.str();
- EXPECT_EQ(string(buff.data(), actual_len), string(str, strlen(str))) << desc.str();
+ string buff = cust_dv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << desc.str();
+ EXPECT_LE(buff.length(), dt_ctx.fmt_out_len) << desc.str();
+ EXPECT_EQ(buff, string(str, strlen(str))) << desc.str();
}
};
diff --git a/be/src/runtime/date-value.cc b/be/src/runtime/date-value.cc
index 13cf2ce..8554242 100644
--- a/be/src/runtime/date-value.cc
+++ b/be/src/runtime/date-value.cc
@@ -27,6 +27,8 @@
namespace impala {
using datetime_parse_util::DateTimeFormatContext;
+using datetime_parse_util::GetMonthAndDayFromDaysSinceJan1;
+using datetime_parse_util::IsLeapYear;
const int EPOCH_YEAR = 1970;
const int MIN_YEAR = 0;
@@ -42,14 +44,6 @@ const int32_t DateValue::MAX_DAYS_SINCE_EPOCH =
const DateValue DateValue::MIN_DATE(MIN_DAYS_SINCE_EPOCH);
const DateValue DateValue::MAX_DATE(MAX_DAYS_SINCE_EPOCH);
-// Describes ranges for months in a non-leap year expressed as number of days since
-// January 1.
-const vector<int> MONTH_RANGES = {
- 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 };
-// Describes ranges for months in a leap year expressed as number of days since January 1.
-const vector<int> LEAP_YEAR_MONTH_RANGES = {
- 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 };
-
DateValue::DateValue(int64_t year, int64_t month, int64_t day)
: days_since_epoch_(INVALID_DAYS_SINCE_EPOCH) {
DCHECK(!IsValid());
@@ -64,25 +58,33 @@ DateValue::DateValue(int64_t year, int64_t month, int64_t day)
}
}
-DateValue DateValue::Parse(const char* str, int len, bool accept_time_toks) {
+DateValue DateValue::ParseSimpleDateFormat(const char* str, int len,
+ bool accept_time_toks) {
DateValue dv;
- discard_result(DateParser::Parse(str, len, accept_time_toks, &dv));
+ discard_result(DateParser::ParseSimpleDateFormat(str, len, accept_time_toks, &dv));
return dv;
}
-DateValue DateValue::Parse(const string& str, bool accept_time_toks) {
- return Parse(str.c_str(), str.size(), accept_time_toks);
+DateValue DateValue::ParseSimpleDateFormat(const string& str, bool accept_time_toks) {
+ return ParseSimpleDateFormat(str.c_str(), str.size(), accept_time_toks);
}
-DateValue DateValue::Parse(const char* str, int len,
+DateValue DateValue::ParseSimpleDateFormat(const char* str, int len,
const DateTimeFormatContext& dt_ctx) {
DateValue dv;
- discard_result(DateParser::Parse(str, len, dt_ctx, &dv));
+ discard_result(DateParser::ParseSimpleDateFormat(str, len, dt_ctx, &dv));
+ return dv;
+}
+
+DateValue DateValue::ParseIsoSqlFormat(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx) {
+ DateValue dv;
+ discard_result(DateParser::ParseIsoSqlFormat(str, len, dt_ctx, &dv));
return dv;
}
-int DateValue::Format(const DateTimeFormatContext& dt_ctx, int len, char* buff) const {
- return DateParser::Format(dt_ctx, *this, len, buff);
+string DateValue::Format(const DateTimeFormatContext& dt_ctx) const {
+ return DateParser::Format(dt_ctx, *this);
}
namespace {
@@ -98,10 +100,6 @@ inline int32_t CalcFirstDayOfYearSinceEpoch(int year) {
+ ((year - EPOCH_YEAR / 400 * 400 + ((m400 != 0) ? 400 - m400 : 0)) / 400 - 1);
}
-inline bool IsLeapYear(int year) {
- return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0));
-}
-
}
bool DateValue::ToYear(int* year) const {
@@ -193,19 +191,7 @@ bool DateValue::ToYearMonthDay(int* year, int* month, int* day) const {
// Day of year. 0 is used for January 1.
int days_since_jan1 = days_since_epoch_ - jan1_dse;
- // Calculate month using month ranges and the average month length.
- const vector<int>& month_ranges = IsLeapYear(*year) ? LEAP_YEAR_MONTH_RANGES
- : MONTH_RANGES;
- int m = static_cast<int>(days_since_jan1 / 30.5);
- DCHECK(month_ranges[m] <= days_since_jan1);
-
- *month = (month_ranges[m + 1] <= days_since_jan1) ? m + 2 : m + 1;
- DCHECK(*month >= 1 && *month <= 12);
-
- // Calculate day.
- *day = days_since_jan1 - month_ranges[*month - 1] + 1;
- DCHECK(*day >= 1 && *day <= 31);
- return true;
+ return GetMonthAndDayFromDaysSinceJan1(*year, days_since_jan1, month, day);
}
int DateValue::WeekDay() const {
diff --git a/be/src/runtime/date-value.h b/be/src/runtime/date-value.h
index b7f2b8a..01692c7 100644
--- a/be/src/runtime/date-value.h
+++ b/be/src/runtime/date-value.h
@@ -156,18 +156,16 @@ class DateValue {
/// Constructors that parse from a date string. See DateParser for details about the
/// date format.
- static DateValue Parse(const char* str, int len, bool accept_time_toks);
- static DateValue Parse(const std::string& str, bool accept_time_toks);
- static DateValue Parse(const char* str, int len,
+ static DateValue ParseSimpleDateFormat(const char* str, int len, bool accept_time_toks);
+ static DateValue ParseSimpleDateFormat(const std::string& str, bool accept_time_toks);
+ static DateValue ParseSimpleDateFormat(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx);
+ static DateValue ParseIsoSqlFormat(const char* str, int len,
const datetime_parse_util::DateTimeFormatContext& dt_ctx);
- /// Format the date using the given 'dt_ctx' format context. The result is placed in
- /// 'buff' buffer which has the size of 'len'. The size of the buffer should be at least
- /// dt_ctx.fmt_out_len + 1. A string terminator will be appended to the string.
- /// Return the number of characters copied in to the buffer (excluding terminator).
- /// If *this is invalid, nothing is placed in 'buff' and -1 is returned.
- int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx, int len,
- char* buff) const;
+ /// Format the date using the given 'dt_ctx' format context. If *this is invalid
+ /// returns an empty string.
+ std::string Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx) const;
bool operator==(const DateValue& other) const {
return days_since_epoch_ == other.days_since_epoch_;
diff --git a/be/src/runtime/datetime-iso-sql-format-parser.cc b/be/src/runtime/datetime-iso-sql-format-parser.cc
new file mode 100644
index 0000000..d0c0547
--- /dev/null
+++ b/be/src/runtime/datetime-iso-sql-format-parser.cc
@@ -0,0 +1,286 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/datetime-iso-sql-format-parser.h"
+
+#include <boost/algorithm/string.hpp>
+#include <cmath>
+
+#include "common/names.h"
+#include "runtime/datetime-iso-sql-format-tokenizer.h"
+#include "util/string-parser.h"
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+bool IsoSqlFormatParser::ParseDateTime(const char* input_str, int input_len,
+ const DateTimeFormatContext& dt_ctx, DateTimeParseResult* result) {
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(result != nullptr);
+ DCHECK(result->hour == 0);
+ if (input_str == nullptr || input_len <= 0) return false;
+
+ int day_in_year = -1;
+
+ const char* current_pos = input_str;
+ const char* end_pos = input_str + input_len;
+ for (int i = 0; i < dt_ctx.toks.size(); ++i) {
+ if (current_pos >= end_pos) return false;
+ const DateTimeFormatToken* tok = &dt_ctx.toks[i];
+ if (tok->type == SEPARATOR) {
+ bool res = ProcessSeparators(¤t_pos, end_pos, dt_ctx, &i);
+ if (!res || current_pos == end_pos) return res;
+ DCHECK(i < dt_ctx.toks.size());
+ // Next token, following the separator sequence.
+ tok = &dt_ctx.toks[i];
+ }
+
+ const char* token_end_pos = FindEndOfToken(current_pos, end_pos - current_pos, *tok);
+ if (token_end_pos == nullptr) return false;
+ int token_len = token_end_pos - current_pos;
+
+ switch(tok->type) {
+ case YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 9999, &result->year)) {
+ return false;
+ }
+ if (token_len < 4) {
+ PrefixYearFromCurrentYear(token_len, dt_ctx.current_time, result);
+ }
+ break;
+ }
+ case ROUND_YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 9999, &result->year)) {
+ return false;
+ }
+ if (token_len == 2) GetRoundYear(dt_ctx.current_time, result);
+ if (token_len == 3 || token_len == 1) {
+ PrefixYearFromCurrentYear(token_len, dt_ctx.current_time, result);
+ }
+ break;
+ }
+ case MONTH_IN_YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 1, 12, &result->month)) {
+ return false;
+ }
+ break;
+ }
+ case DAY_IN_MONTH: {
+ if (!ParseAndValidate(current_pos, token_len, 1, 31, &result->day)) return false;
+ break;
+ }
+ case DAY_IN_YEAR: {
+ if (!ParseAndValidate(current_pos, token_len, 1, 366, &day_in_year)) return false;
+ // Can't figure out the value of MONTH_IN_YEAR and DAY_IN_MONTH here as YEAR
+ // token is also required for that and it might come later in the input string.
+ break;
+ }
+ case HOUR_IN_HALF_DAY: {
+ int hour;
+ if (!ParseAndValidate(current_pos, token_len, 1, 12, &hour)) return false;
+ if (hour == 12) hour = 0;
+ // Note the addition instead of assignment here. PM and HOUR_IN_HALF_DAY can be
+ // in any order in the format token list and PM might add another 12 hours.
+ result->hour += hour;
+ break;
+ }
+ case HOUR_IN_DAY: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 23, &result->hour)) return false;
+ break;
+ }
+ case MINUTE_IN_HOUR: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 59, &result->minute)) {
+ return false;
+ }
+ break;
+ }
+ case SECOND_IN_MINUTE: {
+ if (!ParseAndValidate(current_pos, token_len, 0, 59, &result->second)) {
+ return false;
+ }
+ break;
+ }
+ case SECOND_IN_DAY: {
+ int second_in_day;
+ if (!ParseAndValidate(current_pos, token_len, 0, 86399, &second_in_day)) {
+ return false;
+ }
+ result->second = second_in_day % 60;
+ int minutes_in_day = second_in_day / 60;
+ result->minute = minutes_in_day % 60;
+ result->hour = minutes_in_day / 60;
+ break;
+ }
+ case FRACTION: {
+ if (!ParseFractionToken(current_pos, token_len, result)) return false;
+ break;
+ }
+ case MERIDIEM_INDICATOR: {
+ // Input has already been validated in ParseMeridiemIndicatorFromInput().
+ string indicator(current_pos, token_len);
+ boost::to_upper(indicator);
+ if (indicator == "PM" || indicator == "P.M.") result->hour += 12;
+ break;
+ }
+ case TIMEZONE_HOUR: {
+ // Deliberately ignore the timezone offsets.
+ int dummy_result;
+ if (!ParseAndValidate(current_pos, token_len, -99, 99, &dummy_result)) {
+ return false;
+ }
+ break;
+ }
+ case TIMEZONE_MIN: {
+ // Deliberately ignore the timezone offsets.
+ int dummy_result;
+ if (!ParseAndValidate(current_pos, token_len, 0, 99, &dummy_result)) {
+ return false;
+ }
+ break;
+ }
+ case ISO8601_TIME_INDICATOR:
+ case ISO8601_ZULU_INDICATOR: {
+ DCHECK(token_len == 1);
+ if (toupper(*current_pos) != toupper(*tok->val)) return false;
+ break;
+ }
+ default: {
+ return false;
+ }
+ }
+ current_pos = token_end_pos;
+ }
+
+ // If the format string is over but there are tokens left in the input.
+ if (current_pos < end_pos) return false;
+
+ // Get month and day values from "day in year" and year tokens
+ if (day_in_year != -1) {
+ DCHECK(result->year >= 0 && result->year <= 9999);
+ if (!GetMonthAndDayFromDaysSinceJan1(result->year, day_in_year - 1, &result->month,
+ &result->day)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool IsoSqlFormatParser::ProcessSeparators(const char** current_pos, const char* end_pos,
+ const DateTimeFormatContext& dt_ctx, int* current_tok_idx) {
+ DCHECK(current_pos != nullptr && *current_pos != nullptr);
+ DCHECK(end_pos != nullptr);
+ DCHECK(current_tok_idx != nullptr && *current_tok_idx < dt_ctx.toks.size());
+ if (!IsoSqlFormatTokenizer::IsSeparator(**current_pos)) return false;
+ // Advance to the end of the separator sequence.
+ ++(*current_pos);
+ while (*current_pos < end_pos && IsoSqlFormatTokenizer::IsSeparator(**current_pos)) {
+ ++(*current_pos);
+ }
+ // Advance to the end of the separator sequence in the expected tokens list.
+ ++(*current_tok_idx);
+ while (*current_tok_idx < dt_ctx.toks.size() &&
+ dt_ctx.toks[*current_tok_idx].type == SEPARATOR) {
+ ++(*current_tok_idx);
+ }
+
+ // If we reached the end of input or the end of token sequence, we can return.
+ if (*current_pos >= end_pos || *current_tok_idx >= dt_ctx.toks.size()) {
+ return (*current_pos >= end_pos && *current_tok_idx >= dt_ctx.toks.size());
+ }
+
+ // The last '-' of a separator sequence might be taken as a sign for timezone hour.
+ if (*(*current_pos - 1) == '-' && dt_ctx.toks[*current_tok_idx].type == TIMEZONE_HOUR) {
+ --(*current_pos);
+ }
+ return true;
+}
+
+const char* IsoSqlFormatParser::FindEndOfToken(const char* input_str,
+ int input_len, const DateTimeFormatToken& tok) {
+ DCHECK(input_str != nullptr);
+ DCHECK(input_len >= 0);
+
+ if (input_len == 0) return nullptr;
+
+ // Handle separately the meridiem indicators for two reasons.
+ // 1: They might contain '.' that is not meant to be a separator character.
+ // 2: The length of the token in the pattern might differ from the length of the token
+ // in the input. E.g. "AM" should match with "P.M.".
+ if (tok.type == MERIDIEM_INDICATOR) {
+ return ParseMeridiemIndicatorFromInput(input_str, input_len);
+ }
+
+ int max_tok_len = min(input_len, tok.len);
+ const char* start_of_token = input_str;
+ if (tok.type == TIMEZONE_HOUR) {
+ if (max_tok_len > 2) max_tok_len = 2;
+ if (*start_of_token == '-' || *start_of_token == '+') {
+ ++start_of_token;
+ if (input_len - 1 < max_tok_len) --max_tok_len;
+ }
+ }
+
+ const char* end_pos = start_of_token;
+ while (end_pos < start_of_token + max_tok_len &&
+ !IsoSqlFormatTokenizer::IsSeparator(*end_pos)) {
+ ++end_pos;
+ }
+ if (end_pos == input_str) return nullptr;
+ return end_pos;
+}
+
+const char* IsoSqlFormatParser::ParseMeridiemIndicatorFromInput(
+ const char* input_str, int input_len) {
+ DCHECK(input_str != nullptr);
+ if (input_len >= 4 &&
+ (strncasecmp(input_str, AM_LONG.first, 4) == 0 ||
+ strncasecmp(input_str, PM_LONG.first, 4) == 0 )) {
+ return input_str + 4;
+ }
+ if (input_len >= 2 &&
+ (strncasecmp(input_str, AM.first, 2) == 0 ||
+ strncasecmp(input_str, PM.first, 2) == 0 )) {
+ return input_str + 2;
+ }
+ return nullptr;
+}
+
+void IsoSqlFormatParser::PrefixYearFromCurrentYear(int actual_token_len,
+ const TimestampValue* now, DateTimeParseResult* result) {
+ DCHECK(actual_token_len > 0 && actual_token_len < 4);
+ DCHECK(now != nullptr);
+ DCHECK(result != nullptr);
+ int adjust_factor = pow(10, actual_token_len);
+ int adjustment = (now->date().year() / adjust_factor) * adjust_factor;
+ result->year += adjustment;
+}
+
+void IsoSqlFormatParser::GetRoundYear(const TimestampValue* now,
+ DateTimeParseResult* result) {
+ DCHECK(now != nullptr);
+ DCHECK(result != nullptr);
+ DCHECK(result->year >= 0 && result->year < 100);
+ int postfix_of_curr_year = now->date().year() % 100;
+ if (result->year < 50 && postfix_of_curr_year > 49) result->year += 100;
+ if (result->year > 49 && postfix_of_curr_year < 50) result->year -= 100;
+ result->year += (now->date().year() / 100) * 100;
+}
+
+}
+}
diff --git a/be/src/runtime/datetime-iso-sql-format-parser.h b/be/src/runtime/datetime-iso-sql-format-parser.h
new file mode 100644
index 0000000..b0454d1
--- /dev/null
+++ b/be/src/runtime/datetime-iso-sql-format-parser.h
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "runtime/datetime-parser-common.h"
+
+#include "gutil/macros.h"
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+/// This class is responsible for parsing an input string into a timestamp using
+/// ISO:SQL:2016 datetime format patterns. The list of tokens for the parsing comes as a
+/// constructor parameter in 'dt_ctx' pre-processed by IsoSqlFomratTokenizer.
+/// For more details of the parsing logic please see the design document attached to
+/// IMPALA-4018.
+class IsoSqlFormatParser {
+public:
+ /// Given a list of format tokens in 'dt_ctx' runs through 'input_str' and parses it
+ /// into 'result'. Return value indicates if the parsing was successful.
+ static bool ParseDateTime(const char* input_str, int input_len,
+ const DateTimeFormatContext& dt_ctx, DateTimeParseResult* result)
+ WARN_UNUSED_RESULT;
+
+private:
+ /// 'input_str' points to a location in the input string where the parsing stands now.
+ /// Given 'tok' as the next token in the list of tokens created by the tokenizer this
+ /// functions finds the end of the next token.
+ /// 'input_len' is used for stopping when we reach the end of the input string.
+ /// Return a pointer pointing one position after the last character of the found token.
+ /// If can't identify the next token then returns nullptr.
+ static const char* FindEndOfToken(const char* input_str, int input_len,
+ const DateTimeFormatToken& tok) WARN_UNUSED_RESULT;
+
+ /// Has to call this function when 'input_str' points to the fist character of a
+ /// meridiem indicator. Identifies the last position of a meridiem indicator and returns
+ /// a pointer to the next position after this. If no meridiem indicator is found
+ /// starting from 'input_str' then returns nullptr.
+ static const char* ParseMeridiemIndicatorFromInput(const char* input_str,
+ int input_len);
+
+ /// If the year part of the input is shorter than 4 digits then prefixes the year with
+ /// digits from the current year. Puts the result into 'result->year'.
+ static void PrefixYearFromCurrentYear(int actual_token_len, const TimestampValue* now,
+ DateTimeParseResult* result);
+
+ /// Uses 'result->year' as an input. Can call this function for 2-digit years and it
+ /// constructs a 4-digit year based on the year provided and the current date. Puts the
+ /// result back to 'result->year'.
+ static void GetRoundYear(const TimestampValue* now, DateTimeParseResult* result);
+
+ /// Gets a pointer to the current char in the input string and an index to the current
+ /// token being processed within 'dt_ctx->toks'. Advances these pointers to the end of
+ /// the current separator sequence no matter how long these sequences are. It's expected
+ /// that '**tok' is of type SEPARATOR. Returns false if '**current_pos' is not a
+ /// separator or if either the input ends while having remaining items in 'dt_ctx->toks'
+ /// or the other way around.
+ static bool ProcessSeparators(const char** current_pos, const char* end_pos,
+ const DateTimeFormatContext& dt_ctx, int* dt_ctx_it);
+};
+
+}
+}
diff --git a/be/src/runtime/datetime-iso-sql-format-tokenizer.cc b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
new file mode 100644
index 0000000..07c8cc8
--- /dev/null
+++ b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/datetime-iso-sql-format-tokenizer.h"
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+using std::min;
+using std::string;
+using std::unordered_map;
+using std::unordered_set;
+
+const unordered_map<string, IsoSqlFormatTokenizer::TokenItem>
+ IsoSqlFormatTokenizer::VALID_TOKENS({
+ {"YYYY", IsoSqlFormatTokenizer::TokenItem(YEAR, true, false)},
+ {"YYY", IsoSqlFormatTokenizer::TokenItem(YEAR, true, false)},
+ {"YY", IsoSqlFormatTokenizer::TokenItem(YEAR, true, false)},
+ {"Y", IsoSqlFormatTokenizer::TokenItem(YEAR, true, false)},
+ {"RRRR", IsoSqlFormatTokenizer::TokenItem(ROUND_YEAR, true, false)},
+ {"RR", IsoSqlFormatTokenizer::TokenItem(ROUND_YEAR, true, false)},
+ {"MM", IsoSqlFormatTokenizer::TokenItem(MONTH_IN_YEAR, true, false)},
+ {"DDD", IsoSqlFormatTokenizer::TokenItem(DAY_IN_YEAR, true, false)},
+ {"DD", IsoSqlFormatTokenizer::TokenItem(DAY_IN_MONTH, true, false)},
+ {"HH", IsoSqlFormatTokenizer::TokenItem(HOUR_IN_HALF_DAY, false, true)},
+ {"HH12", IsoSqlFormatTokenizer::TokenItem(HOUR_IN_HALF_DAY, false, true)},
+ {"HH24", IsoSqlFormatTokenizer::TokenItem(HOUR_IN_DAY, false, true)},
+ {"MI", IsoSqlFormatTokenizer::TokenItem(MINUTE_IN_HOUR, false, true)},
+ {"SS", IsoSqlFormatTokenizer::TokenItem(SECOND_IN_MINUTE, false, true)},
+ {"SSSSS", IsoSqlFormatTokenizer::TokenItem(SECOND_IN_DAY, false, true)},
+ {"FF", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF1", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF2", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF3", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF4", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF5", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF6", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF7", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF8", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"FF9", IsoSqlFormatTokenizer::TokenItem(FRACTION, false, true)},
+ {"AM", IsoSqlFormatTokenizer::TokenItem(MERIDIEM_INDICATOR, false, true)},
+ {"A.M.", IsoSqlFormatTokenizer::TokenItem(MERIDIEM_INDICATOR, false, true)},
+ {"PM", IsoSqlFormatTokenizer::TokenItem(MERIDIEM_INDICATOR, false, true)},
+ {"P.M.", IsoSqlFormatTokenizer::TokenItem(MERIDIEM_INDICATOR, false, true)},
+ {"TZH", IsoSqlFormatTokenizer::TokenItem(TIMEZONE_HOUR, false, true)},
+ {"TZM", IsoSqlFormatTokenizer::TokenItem(TIMEZONE_MIN, false, true)},
+ {"T", IsoSqlFormatTokenizer::TokenItem(ISO8601_TIME_INDICATOR, false, true)},
+ {"Z", IsoSqlFormatTokenizer::TokenItem(ISO8601_ZULU_INDICATOR, false, true)}
+});
+
+const unordered_map<string, int> IsoSqlFormatTokenizer::SPECIAL_LENGTHS({
+ {"HH", 2}, {"HH12", 2}, {"HH24", 2}, {"FF", 9}, {"FF1", 1}, {"FF2", 2}, {"FF3", 3},
+ {"FF4", 4}, {"FF5", 5}, {"FF6", 6}, {"FF7", 7}, {"FF8", 8}, {"FF9", 9}, {"TZM", 2}
+});
+
+const unsigned IsoSqlFormatTokenizer::MAX_TOKEN_SIZE = 5;
+
+const int IsoSqlFormatTokenizer::MAX_FORMAT_LENGTH = 100;
+
+FormatTokenizationResult IsoSqlFormatTokenizer::Tokenize() {
+ DCHECK(dt_ctx_ != NULL);
+ DCHECK(dt_ctx_->fmt != NULL);
+ DCHECK(dt_ctx_->fmt_len > 0);
+ DCHECK(dt_ctx_->toks.size() == 0);
+ DCHECK(used_tokens_.empty());
+ if (dt_ctx_->fmt_len > MAX_FORMAT_LENGTH) return TOO_LONG_FORMAT_ERROR;
+ const char* str_end = dt_ctx_->fmt + dt_ctx_->fmt_len;
+ const char* current_pos = dt_ctx_->fmt;
+ while (current_pos < str_end) {
+ ProcessSeparators(¤t_pos);
+ if (current_pos == str_end) break;
+ FormatTokenizationResult parse_result = ProcessNextToken(¤t_pos);
+ if (parse_result != SUCCESS) return parse_result;
+ }
+ if (dt_ctx_->toks.empty()) return GENERAL_ERROR;
+ return CheckIncompatibilities();
+}
+
+void IsoSqlFormatTokenizer::ProcessSeparators(const char** current_pos) {
+ DCHECK(current_pos != nullptr && *current_pos != nullptr);
+ const char* str_begin = dt_ctx_->fmt;
+ const char* str_end = dt_ctx_->fmt + dt_ctx_->fmt_len;
+ while (*current_pos < str_end && IsSeparator(**current_pos)) {
+ dt_ctx_->toks.push_back(DateTimeFormatToken(SEPARATOR, *current_pos - str_begin, 1,
+ *current_pos));
+ ++(*current_pos);
+ }
+}
+
+FormatTokenizationResult IsoSqlFormatTokenizer::ProcessNextToken(
+ const char** current_pos) {
+ DCHECK(*current_pos != nullptr);
+ const char* str_begin = dt_ctx_->fmt;
+ const char* str_end = dt_ctx_->fmt + dt_ctx_->fmt_len;
+ DCHECK(*current_pos < str_end);
+ unsigned curr_token_size =
+ min(MAX_TOKEN_SIZE, static_cast<unsigned>(str_end - *current_pos));
+ string token_to_probe(*current_pos, curr_token_size);
+ boost::to_upper(token_to_probe);
+ while (curr_token_size > 0) {
+ token_to_probe.resize(curr_token_size);
+ const auto token = VALID_TOKENS.find(token_to_probe);
+ if (token != VALID_TOKENS.end()) {
+ if (cast_mode_ == PARSE && IsUsedToken(token_to_probe)) return DUPLICATE_FORMAT;
+ if (!accept_time_toks_ && token->second.time_token) return DATE_WITH_TIME_ERROR;
+ dt_ctx_->toks.push_back(DateTimeFormatToken(token->second.type,
+ *current_pos - str_begin, GetMaxTokenLength(token->first), *current_pos));
+ dt_ctx_->has_date_toks |= token->second.date_token;
+ dt_ctx_->has_time_toks |= token->second.time_token;
+ used_tokens_.insert(token_to_probe);
+ *current_pos += curr_token_size;
+ return SUCCESS;
+ }
+ --curr_token_size;
+ }
+ return GENERAL_ERROR;
+}
+
+int IsoSqlFormatTokenizer::GetMaxTokenLength(const string& token) const {
+ const auto length_it = SPECIAL_LENGTHS.find(token);
+ if (length_it != SPECIAL_LENGTHS.end()) return length_it->second;
+ return token.length();
+}
+
+bool IsoSqlFormatTokenizer::IsUsedToken(const string& token) const {
+ return used_tokens_.find(token) != used_tokens_.end();
+}
+
+bool IsoSqlFormatTokenizer::IsMeridiemIndicatorProvided() const {
+ return IsUsedToken("AM") || IsUsedToken("A.M.") || IsUsedToken("PM") ||
+ IsUsedToken("P.M.");
+}
+
+FormatTokenizationResult IsoSqlFormatTokenizer::CheckIncompatibilities() const {
+ if (cast_mode_ == FORMAT) {
+ if (IsUsedToken("TZH") || IsUsedToken("TZM")) {
+ return TIMEZONE_OFFSET_NOT_ALLOWED_ERROR;
+ }
+ return SUCCESS;
+ }
+
+ short provided_year_count = IsUsedToken("YYYY") + IsUsedToken("YYY") +
+ IsUsedToken("YY") + IsUsedToken("Y");
+ short provided_round_year_count = IsUsedToken("RRRR") + IsUsedToken("RR");
+ if (provided_year_count > 1 || provided_round_year_count > 1) {
+ return CONFLICTING_YEAR_TOKENS_ERROR;
+ }
+
+ if (provided_year_count == 1 && provided_round_year_count == 1) {
+ return YEAR_WITH_ROUNDED_YEAR_ERROR;
+ }
+
+ if (IsUsedToken("DDD") && (IsUsedToken("DD") || IsUsedToken("MM"))) {
+ return DAY_OF_YEAR_TOKEN_CONFLICT;
+ }
+
+ short provided_hour_tokens = IsUsedToken("HH") + IsUsedToken("HH12") +
+ IsUsedToken("HH24");
+ if (provided_hour_tokens > 1) {
+ return CONFLICTING_HOUR_TOKENS_ERROR;
+ }
+
+ short provided_median_tokens = IsUsedToken("AM") + IsUsedToken("A.M.") +
+ IsUsedToken("PM") + IsUsedToken("P.M.");
+ if (provided_median_tokens > 1) {
+ return CONFLICTING_MERIDIEM_TOKENS_ERROR;
+ }
+
+ if (IsMeridiemIndicatorProvided() && IsUsedToken("HH24")) {
+ return MERIDIEM_CONFLICTS_WITH_HOUR_ERROR;
+ }
+
+ if (IsMeridiemIndicatorProvided() && !IsUsedToken("HH") && !IsUsedToken("HH12")) {
+ return MISSING_HOUR_TOKEN_ERROR;
+ }
+
+ if (IsUsedToken("SSSSS") &&
+ (IsUsedToken("HH") || IsUsedToken("HH12") || IsUsedToken("HH24") ||
+ IsUsedToken("MI") || IsUsedToken("SS") || IsMeridiemIndicatorProvided())) {
+ return SECOND_IN_DAY_CONFLICT;
+ }
+
+ if (IsUsedToken("TZM") && !IsUsedToken("TZH")) return MISSING_TZH_TOKEN_ERROR;
+
+ short provided_fractional_second_count = IsUsedToken("FF") + IsUsedToken("FF1") +
+ IsUsedToken("FF2") + IsUsedToken("FF3") + IsUsedToken("FF4") + IsUsedToken("FF5") +
+ IsUsedToken("FF6") + IsUsedToken("FF7") + IsUsedToken("FF8") + IsUsedToken("FF9");
+ if (provided_fractional_second_count > 1) {
+ return CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR;
+ }
+ return SUCCESS;
+}
+
+bool IsoSqlFormatTokenizer::IsSeparator(char c) {
+ return c == '-' || c == ':' || c== ' ' || c == '.' || c == '/' || c== ',' ||
+ c == '\'' || c == ';' ;
+}
+
+}
+}
diff --git a/be/src/runtime/datetime-iso-sql-format-tokenizer.h b/be/src/runtime/datetime-iso-sql-format-tokenizer.h
new file mode 100644
index 0000000..25e2783
--- /dev/null
+++ b/be/src/runtime/datetime-iso-sql-format-tokenizer.h
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "runtime/datetime-parser-common.h"
+
+#include <string>
+#include <unordered_set>
+#include <unordered_map>
+
+#include "gutil/macros.h"
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+/// This class is responsible for splitting a datetime format string into tokens following
+/// the ISO:SQL:2016 standard. For acceptable tokens please see the design document
+/// attached to IMPALA-4018. You can also get a good impression of the available tokens by
+/// checking VALID_TOKENS member and IsSeparator() function of this class.
+class IsoSqlFormatTokenizer {
+public:
+ IsoSqlFormatTokenizer(DateTimeFormatContext* dt_ctx, CastDirection cast_mode,
+ bool time_toks) : dt_ctx_(dt_ctx), cast_mode_(cast_mode),
+ accept_time_toks_(time_toks) {}
+
+ void Reset(DateTimeFormatContext* dt_ctx, CastDirection cast_mode, bool time_toks) {
+ dt_ctx_ = dt_ctx;
+ cast_mode_ = cast_mode;
+ accept_time_toks_ = time_toks;
+ used_tokens_.clear();
+ }
+
+ /// Performs parsing of 'dt_ctx_.fmt' format string. During the process this populates
+ /// 'dt_ctx_' with metadata about the format string such as tokens found in the string.
+ FormatTokenizationResult Tokenize();
+
+ /// Returns true if 'c' is a valid separator
+ static bool IsSeparator(char c);
+private:
+ /// Stores metadata about a specific token type.
+ struct TokenItem {
+ TokenItem(DateTimeFormatTokenType t, bool dt, bool tt) :
+ type(t), date_token(dt), time_token(tt) {}
+
+ DateTimeFormatTokenType type;
+ bool date_token;
+ bool time_token;
+ };
+
+ /// This holds all the valid tokens accepted by this parser. When a format string is
+ /// being parsed the substrings are checked against this member to decide if they are
+ /// valid tokens or not.
+ /// Note, token matching is case-insensitive even though this member contains
+ /// upper-case names only.
+ static const std::unordered_map<std::string, TokenItem> VALID_TOKENS;
+
+ /// Keeps the maximum token lengths for tokens where the token length doesn't equal to
+ /// the length of the datetime pattern. E.g. "HH12" is 4 chars long, however expects
+ /// maximum 2 digits as "10".
+ static const std::unordered_map<std::string, int> SPECIAL_LENGTHS;
+
+ /// This has to be in line with the longest string in VALID_TOKENS;
+ static const unsigned MAX_TOKEN_SIZE;
+
+ /// To be on the safe side this introduces a max length limit for the input format
+ /// strings.
+ static const int MAX_FORMAT_LENGTH;
+
+ /// When parsing is in progress this contains the format tokens that we have found in
+ /// the input format string so far.
+ std::unordered_set<std::string> used_tokens_;
+
+ /// The context that is used for the input of the parsing. It is also populated during
+ /// the parsing process. Not owned by this class.
+ DateTimeFormatContext* dt_ctx_;
+
+ /// Decides whether this is a 'datetime to string' or a 'string to datetime' cast.
+ CastDirection cast_mode_;
+
+ bool accept_time_toks_;
+
+ /// Iterates through all the consecutive separator characters from a given pointer
+ /// 'current' and saves them to 'dt_ctx_'.
+ void ProcessSeparators(const char** current);
+
+ /// Identifies the next token using VALID_TOKENS and saves it to 'dt_ctx_'. Finds the
+ /// longest possible match.
+ FormatTokenizationResult ProcessNextToken(const char** current_pos);
+
+ /// Checks if the token has special length in 'SPECIAL_LENGTHS'. If finds a special
+ /// length then returns that, otherwise returns the length of 'token'.
+ int GetMaxTokenLength(const std::string& token) const;
+
+ /// Checks if 'token' is present in 'used_tokens_';
+ bool IsUsedToken(const std::string& token) const;
+
+ /// Checks if any of the meridiem indicators are present in 'used_tokens_'.
+ bool IsMeridiemIndicatorProvided() const;
+
+ /// Checks if the end product of the parsing contains format tokens that collide with
+ /// each other like YYYY and RR. Note, this is mostly relevant for a 'string to
+ /// datetime' conversion. The only restriction for 'datetime to string' path is that
+ /// timezone tokens are not allowed since Impala doesn't store timezone information.
+ FormatTokenizationResult CheckIncompatibilities() const;
+};
+
+}
+}
diff --git a/be/src/runtime/datetime-parse-util.h b/be/src/runtime/datetime-parse-util.h
deleted file mode 100644
index 7a33102..0000000
--- a/be/src/runtime/datetime-parse-util.h
+++ /dev/null
@@ -1,262 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include <vector>
-#include <boost/date_time/gregorian/gregorian.hpp>
-#include <boost/date_time/posix_time/ptime.hpp>
-
-#include "common/logging.h"
-#include "runtime/string-value.inline.h"
-
-namespace impala {
-
-class TimestampValue;
-
-/// This namespace contains helper functions and types to parse default and custom
-/// formatted date-time values. They are used in TimestampParser and DateParser classes.
-namespace datetime_parse_util {
-
-/// Add support for dealing with custom date/time formats in Impala. The following
-/// date/time tokens are supported:
-/// y – Year
-/// M – Month
-/// d – Day
-/// H – Hour
-/// m – Minute
-/// s – second
-/// S – Fractional second
-///
-/// TimeZone offset formats (Must be at the end of format string):
-/// +/-hh:mm
-/// +/-hhmm
-/// +/-hh
-///
-///
-/// The token names and usage have been modeled after the SimpleDateFormat class used in
-/// Java, with only the above list of tokens being supported. All fields will consume
-/// variable length inputs when parsing an input string and must therefore use separators
-/// to specify the boundaries of the fields, with the exception of TimeZone values, which
-/// have to be of fixed width. Repeating tokens can be used to specify fields of exact
-/// witdh, e.g. in yy-MM both fields must be of exactly length two. When using fixed width
-/// fields values must be zero-padded and output values will be zero padded during
-/// formatting. There is one exception to this: a month field of length 3 will specify
-/// literal month names instead of zero padding, i.e., yyyy-MMM-dd will parse from and
-/// format to strings like 2013-Nov-21. When using fields of fixed width the separators
-/// can be omitted.
-///
-///
-/// Formatting character groups can appear in any order along with any separators
-/// except TimeZone offset.
-/// e.g.
-/// yyyy/MM/dd
-/// dd-MMM-yy
-/// (dd)(MM)(yyyy) HH:mm:ss
-/// yyyy-MM-dd HH:mm:ss+hh:mm
-/// ..etc..
-///
-/// The following features are not supported:
-/// Long literal months e.g. MMMM
-/// Nested strings e.g. “Year: “ yyyy “Month: “ mm “Day: “ dd
-/// Lazy formatting
-
-/// Used to indicate the type of a date/time format token group.
-enum DateTimeFormatTokenType {
- UNKNOWN = 0,
- SEPARATOR,
- YEAR,
- MONTH_IN_YEAR,
- /// Indicates a short literal month e.g. MMM (Aug). Note that the month name is case
- /// insensitive for an input scenario and printed in camel case for an output scenario.
- MONTH_IN_YEAR_SLT,
- DAY_IN_MONTH,
- HOUR_IN_DAY,
- MINUTE_IN_HOUR,
- SECOND_IN_MINUTE,
- /// Indicates fractional seconds e.g.14:52:36.2334. By default this provides nanosecond
- /// resolution.
- FRACTION,
- TZ_OFFSET,
-};
-
-/// Used to store metadata about a token group within a date/time format.
-struct DateTimeFormatToken {
- /// Indicates the type of date/time format token e.g. year
- DateTimeFormatTokenType type;
- /// The position of where this token group is supposed to start in the date/time string
- /// to be parsed
- int pos;
- /// The length of the token group
- int len;
- /// A pointer to the date/time format string that is positioned at the start of this
- /// token group
- const char* val;
-
- DateTimeFormatToken(DateTimeFormatTokenType type, int pos, int len, const char* val)
- : type(type),
- pos(pos),
- len(len),
- val(val) {
- }
-};
-
-/// This structure is used to hold metadata for a date/time format. Each token group
-/// within the raw format is parsed and placed in this structure along with other high
-/// level information e.g. if the format contains date and/or time tokens. This context
-/// is used during date/time parsing.
-struct DateTimeFormatContext {
- const char* fmt;
- int fmt_len;
- /// Holds the expanded length of fmt_len plus any required space when short format
- /// tokens are used. The output buffer size is driven from this value. For example, in
- /// an output scenario a user may provide the format yyyy-M-d, if the day and month
- /// equates to 12, 21 then extra space is needed in the buffer to hold the values. The
- /// short format type e.g. yyyy-M-d is valid where no zero padding is required on single
- /// digits.
- int fmt_out_len;
- std::vector<DateTimeFormatToken> toks;
- bool has_date_toks;
- bool has_time_toks;
- /// Current time - 80 years to determine the actual year when
- /// parsing 1 or 2-digit year token.
- boost::posix_time::ptime century_break_ptime;
-
- DateTimeFormatContext() {
- Reset(nullptr);
- }
-
- DateTimeFormatContext(const char* fmt) {
- Reset(fmt);
- }
-
- DateTimeFormatContext(const char* fmt, int fmt_len) {
- Reset(fmt, fmt_len);
- }
-
- /// Set the century break when parsing 1 or 2-digit year format.
- /// When parsing 1 or 2-digit year, the year should be in the interval
- /// [now - 80 years, now + 20 years), according to Hive.
- void SetCenturyBreak(const TimestampValue& now);
-
- void Reset(const char* fmt, int fmt_len) {
- this->fmt = fmt;
- this->fmt_len = fmt_len;
- this->fmt_out_len = fmt_len;
- this->has_date_toks = false;
- this->has_time_toks = false;
- this->toks.clear();
- this->century_break_ptime = boost::posix_time::not_a_date_time;
- }
-
- void Reset(const char* fmt) {
- Reset(fmt, (fmt == nullptr) ? 0 : strlen(fmt));
- }
-};
-
-/// Stores the results of parsing a date/time string.
-struct DateTimeParseResult {
- int year = -1;
- int month = 0;
- int day = 0;
- int hour = 0;
- int minute = 0;
- int second = 0;
- int32_t fraction = 0;
- boost::posix_time::time_duration tz_offset =
- boost::posix_time::time_duration(0, 0, 0, 0);
- // Whether to realign the year for 2-digit year format
- bool realign_year = false;
-};
-
-/// Constants to hold default format lengths.
-const int DEFAULT_DATE_FMT_LEN = 10;
-const int DEFAULT_TIME_FMT_LEN = 8;
-const int DEFAULT_TIME_FRAC_FMT_LEN = 18;
-const int DEFAULT_SHORT_DATE_TIME_FMT_LEN = 19;
-const int DEFAULT_DATE_TIME_FMT_LEN = 29;
-
-/// Pseudo-constant default date/time contexts. Backwards compatibility is provided on
-/// variable length fractional components by defining a format context for each expected
-/// length (0 - 9). This logic will be refactored when the parser supports lazy token
-/// groups.
-extern DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
-extern DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
-extern DateTimeFormatContext DEFAULT_DATE_CTX;
-extern DateTimeFormatContext DEFAULT_TIME_CTX;
-extern DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
-extern DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
-extern DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
-
-/// Initialize the static parser context which includes default date/time formats and
-/// lookup tables. This *must* be called before any of the Parse* related functions can
-/// be used.
-void InitParseCtx();
-
-/// Return true iff InitParseCtx() has been called.
-bool IsParseCtxInitialized();
-
-/// Parse the date/time format into tokens and place them in the context.
-/// dt_ctx -- date/time format context
-/// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
-/// rejected.
-/// Return true if the parse was successful.
-bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks = true);
-
-/// Parse the date/time string to generate the DateTimeFormatToken required by
-/// DateTimeFormatContext. Similar to ParseFormatTokens() this function will take the
-/// string and length, then heuristically determine whether the value contains date
-/// tokens, time tokens, or both. Unlike ParseFormatTokens, it does not require the
-/// template format string.
-/// dt_ctx -- date/time format context (must contain valid tokens)
-/// accept_time_toks -- if true, time tokens are accepted, otherwise time tokens are
-/// rejected.
-/// accept_time_toks_only -- if true, time tokens w/o date tokens are accepted. Otherwise,
-/// they are rejected.
-/// Return true if the date/time was successfully parsed.
-bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks = true,
- bool accept_time_toks_only = true);
-
-/// Parse date/time string to find the corresponding default date/time format context. The
-/// string must adhere to a default date/time format.
-/// str -- valid pointer to the string to parse.
-/// len -- length of the string to parse (must be > 0)
-/// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
-/// rejected.
-/// accept_time_toks_only -- if true, time tokens without date tokens are accepted.
-/// Otherwise, they are rejected.
-/// Return the corresponding default format context if parsing succeeded, or nullptr
-/// otherwise.
-const DateTimeFormatContext* ParseDefaultFormatTokensByStr(const char* str, int len,
- bool accept_time_toks, bool accept_time_toks_only);
-
-/// Parse a date/time string. The data must adhere to the format, otherwise it will be
-/// rejected i.e. no missing tokens.
-/// ParseDateTime() does only a basic validation on the parsed date/time values. The
-/// caller is responsible for implementing rigid data validation and range-check.
-/// str -- valid pointer to the string to parse
-/// len -- length of the string to parse (must be > 0)
-/// dt_ctx -- date/time format context (must contain valid tokens)
-/// dt_result -- the struct where the results of the parsing will be placed
-/// Return true if the date/time was successfully parsed.
-bool ParseDateTime(const char* str, int len, const DateTimeFormatContext& dt_ctx,
- DateTimeParseResult* dt_result);
-
-}
-
-}
diff --git a/be/src/runtime/datetime-parser-common.cc b/be/src/runtime/datetime-parser-common.cc
new file mode 100644
index 0000000..514e0d5
--- /dev/null
+++ b/be/src/runtime/datetime-parser-common.cc
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "datetime-parser-common.h"
+
+#include <boost/date_time/gregorian/gregorian.hpp>
+
+#include "runtime/string-value.h"
+#include "util/string-parser.h"
+
+using std::string;
+using std::unordered_set;
+
+namespace impala {
+
+namespace datetime_parse_util {
+
+void DateTimeFormatContext::SetCenturyBreakAndCurrentTime(const TimestampValue& now) {
+ current_time = &now;
+ const boost::gregorian::date& now_date = now.date();
+ // If the century break is at an invalid 02/29, set it to 02/28 for consistency with
+ // Hive.
+ if (now_date.month() == 2 && now_date.day() == 29 &&
+ !boost::gregorian::gregorian_calendar::is_leap_year(now_date.year() - 80)) {
+ century_break_ptime = boost::posix_time::ptime(
+ boost::gregorian::date(now_date.year() - 80, 2, 28), now.time());
+ } else {
+ century_break_ptime = boost::posix_time::ptime(
+ boost::gregorian::date(now_date.year() - 80, now_date.month(), now_date.day()),
+ now.time());
+ }
+ DCHECK(!century_break_ptime.is_special());
+}
+
+void DateTimeFormatContext::Reset(const char* fmt, int fmt_len) {
+ this->fmt = fmt;
+ this->fmt_len = fmt_len;
+ this->fmt_out_len = fmt_len;
+ this->has_date_toks = false;
+ this->has_time_toks = false;
+ this->toks.clear();
+ this->century_break_ptime = boost::posix_time::not_a_date_time;
+ this->current_time = nullptr;
+}
+
+void ReportBadFormat(FunctionContext* context, FormatTokenizationResult error_type,
+ const StringVal& format, bool is_error) {
+ DCHECK(context != nullptr);
+ std::stringstream ss;
+ if (format.is_null || format.len == 0) {
+ ss << "Bad date/time conversion format: format string is NULL or has 0 length";
+ } else {
+ switch (error_type) {
+ case DUPLICATE_FORMAT:
+ ss << "PARSE ERROR: Invalid duplication of format element";
+ break;
+ case YEAR_WITH_ROUNDED_YEAR_ERROR:
+ ss << "PARSE ERROR: Both year and round year are provided";
+ break;
+ case CONFLICTING_YEAR_TOKENS_ERROR:
+ ss << "PARSE ERROR: Multiple year token provided";
+ break;
+ case DAY_OF_YEAR_TOKEN_CONFLICT:
+ ss << "PARSE ERROR: Day of year provided with day or month token";
+ break;
+ case CONFLICTING_HOUR_TOKENS_ERROR:
+ ss << "PARSE ERROR: Multiple hour tokens provided";
+ break;
+ case CONFLICTING_MERIDIEM_TOKENS_ERROR:
+ ss << "PARSE ERROR: Multiple median indicator tokens provided";
+ break;
+ case MERIDIEM_CONFLICTS_WITH_HOUR_ERROR:
+ ss << "PARSE ERROR: Conflict between median indicator and hour token";
+ break;
+ case MISSING_HOUR_TOKEN_ERROR:
+ ss << "PARSE ERROR: Missing hour token";
+ break;
+ case SECOND_IN_DAY_CONFLICT:
+ ss << "PARSE ERROR: Second of day token conflicts with other token(s)";
+ break;
+ case TOO_LONG_FORMAT_ERROR:
+ ss << "PARSE ERROR: The input format is too long";
+ break;
+ case TIMEZONE_OFFSET_NOT_ALLOWED_ERROR:
+ ss << "PARSE ERROR: Timezone offset not allowed in a datetime to string "
+ "conversion";
+ break;
+ case MISSING_TZH_TOKEN_ERROR:
+ ss << "PARSE ERROR: TZH token is required for TZM";
+ break;
+ case DATE_WITH_TIME_ERROR:
+ ss << "PARSE ERROR: Time tokens provided with date type.";
+ break;
+ case CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR:
+ ss << "PARSE ERROR: Multiple fractional second token provided.";
+ break;
+ default:
+ const StringValue& fmt = StringValue::FromStringVal(format);
+ ss << "Bad date/time conversion format: " << fmt.DebugString();
+ }
+ }
+ if (is_error) {
+ context->SetError(ss.str().c_str());
+ } else {
+ context->AddWarning(ss.str().c_str());
+ }
+}
+
+bool ParseAndValidate(const char* token, int token_len, int min, int max,
+ int* result) {
+ DCHECK(token != nullptr);
+ DCHECK(token_len > 0);
+ DCHECK(result != nullptr);
+ StringParser::ParseResult status;
+ *result = StringParser::StringToInt<int>(token, token_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ if (UNLIKELY(*result < min || *result > max)) return false;
+ return true;
+}
+
+bool ParseFractionToken(const char* token, int token_len,
+ DateTimeParseResult* result) {
+ DCHECK(token != nullptr);
+ DCHECK(token_len > 0);
+ DCHECK(result != nullptr);
+ StringParser::ParseResult status;
+ result->fraction =
+ StringParser::StringToInt<int32_t>(token, token_len, &status);
+ if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
+ // A user may specify a time of 04:30:22.1238, the parser will return 1238 for
+ // the fractional portion. This does not represent the intended value of
+ // 123800000, therefore the number must be scaled up.
+ if (token_len < FRACTIONAL_SECOND_MAX_LENGTH) {
+ result->fraction *= std::pow(10, FRACTIONAL_SECOND_MAX_LENGTH - token_len);
+ }
+ return true;
+}
+
+int GetDayInYear(int year, int month, int day_in_month) {
+ DCHECK(month >= 1 && month <= 12);
+ const vector<int>& month_ranges = IsLeapYear(year) ? LEAP_YEAR_MONTH_RANGES :
+ MONTH_RANGES;
+ return day_in_month + month_ranges[month - 1];
+}
+
+bool GetMonthAndDayFromDaysSinceJan1(int year, int days_since_jan1, int* month,
+ int* day) {
+ DCHECK(days_since_jan1 >= 0 && days_since_jan1 < 366);
+ DCHECK(month != nullptr);
+ DCHECK(day != nullptr);
+ // Calculate month using month ranges and the average month length.
+ const vector<int>& month_ranges = IsLeapYear(year) ? LEAP_YEAR_MONTH_RANGES :
+ MONTH_RANGES;
+ int m = static_cast<int>(days_since_jan1 / 30.5);
+ DCHECK(month_ranges[m] <= days_since_jan1);
+
+ *month = (month_ranges[m + 1] <= days_since_jan1) ? m + 2 : m + 1;
+ if (*month < 1 || *month > 12) return false;
+
+ // Calculate day.
+ *day = days_since_jan1 - month_ranges[*month - 1] + 1;
+ return (*day >= 1 && *day <= 31);
+}
+
+}
+}
diff --git a/be/src/runtime/datetime-parser-common.h b/be/src/runtime/datetime-parser-common.h
new file mode 100644
index 0000000..3093c76
--- /dev/null
+++ b/be/src/runtime/datetime-parser-common.h
@@ -0,0 +1,230 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <boost/date_time/posix_time/ptime.hpp>
+#include "gutil/macros.h"
+#include <unordered_set>
+#include <vector>
+
+#include "runtime/timestamp-value.h"
+#include "udf/udf.h"
+
+namespace impala {
+
+using impala_udf::FunctionContext;
+using impala_udf::StringVal;
+
+// Impala provides multiple algorithms to parse datetime formats:
+// - SimpleDateFormat: This is the one that is traditionally used with functions such
+// as to_timestamp() and from_timestamp().
+// - ISO SQL:2016 compliant datetime pattern matching. CAST(..FORMAT..) comes with
+// support for this pattern only.
+// This is a collection of the logic that is shared between the 2 types of pattern
+// matching including result codes, error reporting, format token types etc.
+namespace datetime_parse_util {
+const int MONTH_LENGTHS[12] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+
+const int FRACTIONAL_SECOND_MAX_LENGTH = 9;
+
+// Describes ranges for months in a non-leap year expressed as number of days since
+// January 1.
+const std::vector<int> MONTH_RANGES = {
+ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365 };
+
+// Describes ranges for months in a leap year expressed as number of days since January 1.
+const std::vector<int> LEAP_YEAR_MONTH_RANGES = {
+ 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 };
+
+// Contains all the possible result codes that can come from parsing a datetime format
+// pattern.
+enum FormatTokenizationResult {
+ SUCCESS,
+ GENERAL_ERROR,
+ DUPLICATE_FORMAT,
+ YEAR_WITH_ROUNDED_YEAR_ERROR,
+ CONFLICTING_YEAR_TOKENS_ERROR,
+ DAY_OF_YEAR_TOKEN_CONFLICT,
+ CONFLICTING_HOUR_TOKENS_ERROR,
+ CONFLICTING_MERIDIEM_TOKENS_ERROR,
+ MERIDIEM_CONFLICTS_WITH_HOUR_ERROR,
+ MISSING_HOUR_TOKEN_ERROR,
+ SECOND_IN_DAY_CONFLICT,
+ TOO_LONG_FORMAT_ERROR,
+ TIMEZONE_OFFSET_NOT_ALLOWED_ERROR,
+ MISSING_TZH_TOKEN_ERROR,
+ DATE_WITH_TIME_ERROR,
+ CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR
+};
+
+/// Holds all the token types that serve as building blocks for datetime format patterns.
+enum DateTimeFormatTokenType {
+ UNKNOWN = 0,
+ SEPARATOR,
+ YEAR,
+ ROUND_YEAR,
+ MONTH_IN_YEAR,
+ MONTH_IN_YEAR_SLT,
+ DAY_IN_MONTH,
+ DAY_IN_YEAR,
+ HOUR_IN_DAY,
+ HOUR_IN_HALF_DAY,
+ MINUTE_IN_HOUR,
+ SECOND_IN_DAY,
+ SECOND_IN_MINUTE,
+ FRACTION,
+ TZ_OFFSET,
+ TIMEZONE_HOUR,
+ TIMEZONE_MIN,
+ MERIDIEM_INDICATOR,
+ ISO8601_TIME_INDICATOR,
+ ISO8601_ZULU_INDICATOR
+};
+
+/// Indicates whether the cast is a 'datetime to string' or a 'string to datetime' cast.
+/// PARSE is a string type to datetime type cast.
+/// FORMAT is a datetime type to string type cast.
+enum CastDirection {
+ PARSE,
+ FORMAT
+};
+
+typedef std::pair<const char*, const char*> MERIDIEM_INDICATOR_TEXT;
+const MERIDIEM_INDICATOR_TEXT AM = {"AM", "am"};
+const MERIDIEM_INDICATOR_TEXT AM_LONG = {"A.M.", "a.m."};
+const MERIDIEM_INDICATOR_TEXT PM = {"PM", "pm"};
+const MERIDIEM_INDICATOR_TEXT PM_LONG = {"P.M.", "p.m."};
+
+/// Stores metadata about a token within a datetime format.
+struct DateTimeFormatToken {
+ /// Indicates the type of datetime format token.
+ DateTimeFormatTokenType type;
+ /// The position of where this token is supposed to start in the datetime string
+ /// to be parsed.
+ int pos;
+ /// The length of the token.
+ int len;
+ /// A pointer to the beginning of this token in the format string.
+ const char* val;
+
+ DateTimeFormatToken(DateTimeFormatTokenType type, int pos, int len, const char* val)
+ : type(type),
+ pos(pos),
+ len(len),
+ val(val) {
+ }
+};
+
+/// Holds metadata about the datetime format. In the format parsing process the members of
+/// this struct are populated gradually as the process advances. After the parsing process
+/// this holds the found format tokens alongside with auxiliary information such as
+/// whether the input format contains date or time tokens or both.
+struct DateTimeFormatContext {
+ /// Pointer to the beginning of the format string.
+ const char* fmt;
+ /// Length of the format string.
+ int fmt_len;
+ /// Expected length of the output of a 'datetime to string' cast. This usually equals to
+ /// the length of the input format string. However, there are some edge cases where this
+ /// is not true:
+ /// - SimpleDateFormat parsing on '2019-11-10' as input and 'yyyy-d-m' as format
+ /// produces output that is longer than the format string.
+ /// - ISO SQL parsing has token types where the output length is different from the
+ /// token length like: 'MONTH', 'DAY', 'HH12', 'HH24', FF1, FF2, FF4, etc.
+ int fmt_out_len;
+ /// Vector of tokens found in the format string.
+ std::vector<DateTimeFormatToken> toks;
+ bool has_date_toks;
+ bool has_time_toks;
+ /// Used for casting with SimpleDateFormat to handle rounded year. Make sure you call
+ /// SetCenturyBreakAndCurrentTime() before using this member.
+ boost::posix_time::ptime century_break_ptime;
+ /// Used for round year and less than 4-digit year calculation in ISO:SQL:2016 parsing.
+ /// Make sure you call SetCenturyBreakAndCurrentTime() before using this member. Not
+ /// owned by this object.
+ const TimestampValue* current_time;
+
+ DateTimeFormatContext() {
+ Reset(nullptr);
+ }
+
+ DateTimeFormatContext(const char* fmt) {
+ Reset(fmt);
+ }
+
+ DateTimeFormatContext(const char* fmt, int fmt_len) {
+ Reset(fmt, fmt_len);
+ }
+
+ /// Set the century break for parsing 1 or 2-digit year format. When parsing 1 or
+ /// 2-digit year, the year should be in the interval [now - 80 years, now + 20 years),
+ /// according to Hive. Also sets the current time that is used for round year
+ /// calculation in ISO:SQL:2016 parsing.
+ void SetCenturyBreakAndCurrentTime(const TimestampValue& now);
+
+ /// Initializes all the members of this object.
+ void Reset(const char* fmt, int fmt_len);
+
+ void Reset(const char* fmt) {
+ Reset(fmt, (fmt == nullptr) ? 0 : strlen(fmt));
+ }
+};
+
+/// Stores the results of parsing a date/time string.
+struct DateTimeParseResult {
+ int year = -1;
+ int month = 0;
+ int day = 0;
+ int hour = 0;
+ int minute = 0;
+ int second = 0;
+ int32_t fraction = 0;
+ boost::posix_time::time_duration tz_offset =
+ boost::posix_time::time_duration(0, 0, 0, 0);
+ bool realign_year = false;
+};
+
+/// This function is used to indicate an error or warning when the input format
+/// tokenization fails for some reason. Constructs an error message based on 'error_type'
+/// and pushes it to 'context'. Depending on 'is_error' the message can be an error or
+/// warning.
+void ReportBadFormat(FunctionContext* context, FormatTokenizationResult error_type,
+ const StringVal& format, bool is_error);
+
+bool ParseAndValidate(const char* token, int token_len, int min, int max,
+ int* result) WARN_UNUSED_RESULT;
+
+bool ParseFractionToken(const char* token, int token_len,
+ DateTimeParseResult* result) WARN_UNUSED_RESULT;
+
+inline bool IsLeapYear(int year) {
+ return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
+}
+
+/// Given the year, month and the day in month calculates the day in year using
+/// MONTH_LENGTHS.
+int GetDayInYear(int year, int month, int day_in_month);
+
+/// Gets a year and the number of days passed since 1st of January that year. Calculates
+/// the month and the day of that year. Returns false if any of the in parameters are
+/// invalid e.g. if calling this function with a non-leap year and 'days_since_jan1' is
+/// 365. Returns true on success.
+bool GetMonthAndDayFromDaysSinceJan1(int year, int days_since_jan1, int* month, int* day)
+ WARN_UNUSED_RESULT;
+}
+}
diff --git a/be/src/runtime/datetime-parse-util.cc b/be/src/runtime/datetime-simple-date-format-parser.cc
similarity index 74%
rename from be/src/runtime/datetime-parse-util.cc
rename to be/src/runtime/datetime-simple-date-format-parser.cc
index f534cab..ceeb654 100644
--- a/be/src/runtime/datetime-parse-util.cc
+++ b/be/src/runtime/datetime-simple-date-format-parser.cc
@@ -15,70 +15,44 @@
// specific language governing permissions and limitations
// under the License.
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include <algorithm>
-#include <boost/date_time/gregorian/gregorian.hpp>
-#include <boost/unordered_map.hpp>
-
#include "cctz/civil_time.h"
-#include "exprs/timestamp-functions.h"
-#include "runtime/timestamp-value.h"
-#include "util/string-parser.h"
-
#include "common/names.h"
+#include "runtime/string-value.h"
+#include "runtime/string-value.inline.h"
+#include "util/string-parser.h"
using boost::unordered_map;
-using boost::gregorian::date;
-using boost::gregorian::gregorian_calendar;
-using boost::posix_time::ptime;
using boost::posix_time::time_duration;
namespace impala {
namespace datetime_parse_util {
-void DateTimeFormatContext::SetCenturyBreak(const TimestampValue& now) {
- const date& now_date = now.date();
- // If the century break is at an invalid 02/29, set it to 02/28 for consistency with
- // Hive.
- if (now_date.month() == 2 && now_date.day() == 29 &&
- !gregorian_calendar::is_leap_year(now_date.year() - 80)) {
- century_break_ptime = ptime(date(now_date.year() - 80, 2, 28), now.time());
- } else {
- century_break_ptime = ptime(
- date(now_date.year() - 80, now_date.month(), now_date.day()), now.time());
- }
-}
+bool SimpleDateFormatTokenizer::initialized = false;
+bool SimpleDateFormatParser::initialized = false;
-/// Used to indicate if the parsing state has been initialized.
-bool initialized = false;
+const int SimpleDateFormatTokenizer::DEFAULT_DATE_FMT_LEN = 10;
+const int SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN = 8;
+const int SimpleDateFormatTokenizer::DEFAULT_TIME_FRAC_FMT_LEN = 18;
+const int SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_FMT_LEN = 19;
+const int SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_FMT_LEN = 29;
-/// Lazily initialized pseudo-constant hashmap for mapping month names to an index.
-unordered_map<StringValue, int> REV_MONTH_INDEX;
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_CTX;
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_SHORT_ISO_DATE_TIME_CTX;
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_DATE_CTX;
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_TIME_CTX;
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_CTX[10];
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_ISO_DATE_TIME_CTX[10];
+DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_TIME_FRAC_CTX[10];
-DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
-DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
-DateTimeFormatContext DEFAULT_DATE_CTX;
-DateTimeFormatContext DEFAULT_TIME_CTX;
-DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
-DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
-DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
+unordered_map<StringValue, int> SimpleDateFormatParser::REV_MONTH_INDEX;
-void InitParseCtx() {
+void SimpleDateFormatTokenizer::InitCtx() {
if (initialized) return;
- // This needs to be lazily init'd because a StringValues hash function will be invoked
- // for each entry that's placed in the map. The hash function expects that
- // CpuInfo::Init() has already been called.
- REV_MONTH_INDEX = boost::unordered_map<StringValue, int>({
- {StringValue("jan"), 1}, {StringValue("feb"), 2},
- {StringValue("mar"), 3}, {StringValue("apr"), 4},
- {StringValue("may"), 5}, {StringValue("jun"), 6},
- {StringValue("jul"), 7}, {StringValue("aug"), 8},
- {StringValue("sep"), 9}, {StringValue("oct"), 10},
- {StringValue("nov"), 11}, {StringValue("dec"), 12}
- });
// Setup the default date/time context yyyy-MM-dd HH:mm:ss.SSSSSSSSS
const char* DATE_TIME_CTX_FMT = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS";
@@ -86,47 +60,62 @@ void InitParseCtx() {
for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
DEFAULT_DATE_TIME_CTX[i].Reset(DATE_TIME_CTX_FMT,
DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- ParseFormatTokens(&DEFAULT_DATE_TIME_CTX[i]);
+ Tokenize(&DEFAULT_DATE_TIME_CTX[i]);
}
// Setup the default ISO date/time context yyyy-MM-ddTHH:mm:ss.SSSSSSSSS
for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
DEFAULT_ISO_DATE_TIME_CTX[i].Reset("yyyy-MM-ddTHH:mm:ss.SSSSSSSSS",
DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- ParseFormatTokens(&DEFAULT_ISO_DATE_TIME_CTX[i]);
+ Tokenize(&DEFAULT_ISO_DATE_TIME_CTX[i]);
}
// Setup the short default date/time context yyyy-MM-dd HH:mm:ss
DEFAULT_SHORT_DATE_TIME_CTX.Reset("yyyy-MM-dd HH:mm:ss");
- ParseFormatTokens(&DEFAULT_SHORT_DATE_TIME_CTX);
+ Tokenize(&DEFAULT_SHORT_DATE_TIME_CTX);
// Setup the short default ISO date/time context yyyy-MM-ddTHH:mm:ss
DEFAULT_SHORT_ISO_DATE_TIME_CTX.Reset("yyyy-MM-ddTHH:mm:ss");
- ParseFormatTokens(&DEFAULT_SHORT_ISO_DATE_TIME_CTX);
+ Tokenize(&DEFAULT_SHORT_ISO_DATE_TIME_CTX);
// Setup the default short date context yyyy-MM-dd
DEFAULT_DATE_CTX.Reset("yyyy-MM-dd");
- ParseFormatTokens(&DEFAULT_DATE_CTX);
+ Tokenize(&DEFAULT_DATE_CTX);
// Setup the default short time context HH:mm:ss
DEFAULT_TIME_CTX.Reset("HH:mm:ss");
- ParseFormatTokens(&DEFAULT_TIME_CTX);
+ Tokenize(&DEFAULT_TIME_CTX);
// Setup the default short time context with fractional seconds HH:mm:ss.SSSSSSSSS
for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
DEFAULT_TIME_FRAC_CTX[i].Reset(DATE_TIME_CTX_FMT + 11,
DEFAULT_TIME_FRAC_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- ParseFormatTokens(&DEFAULT_TIME_FRAC_CTX[i]);
+ Tokenize(&DEFAULT_TIME_FRAC_CTX[i]);
}
+
// Flag that the parser is ready.
initialized = true;
}
-bool IsParseCtxInitialized() {
- return initialized;
+void SimpleDateFormatParser::InitCtx() {
+ if (initialized) return;
+ // This needs to be lazily init'd because a StringValues hash function will be invoked
+ // for each entry that's placed in the map. The hash function expects that
+ // CpuInfo::Init() has already been called.
+ REV_MONTH_INDEX = boost::unordered_map<StringValue, int>({
+ {StringValue("jan"), 1}, {StringValue("feb"), 2},
+ {StringValue("mar"), 3}, {StringValue("apr"), 4},
+ {StringValue("may"), 5}, {StringValue("jun"), 6},
+ {StringValue("jul"), 7}, {StringValue("aug"), 8},
+ {StringValue("sep"), 9}, {StringValue("oct"), 10},
+ {StringValue("nov"), 11}, {StringValue("dec"), 12}
+ });
+
+ initialized = true;
}
-bool IsValidTZOffset(const char* str_begin, const char* str_end) {
+bool SimpleDateFormatTokenizer::IsValidTZOffset(const char* str_begin,
+ const char* str_end) {
if (*str_begin == '+' || *str_begin == '-') {
++str_begin;
switch (str_end - str_begin) {
@@ -143,7 +132,8 @@ bool IsValidTZOffset(const char* str_begin, const char* str_end) {
return false;
}
-bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
+bool SimpleDateFormatTokenizer::Tokenize(DateTimeFormatContext* dt_ctx,
+ bool accept_time_toks) {
DCHECK(dt_ctx != NULL);
DCHECK(dt_ctx->fmt != NULL);
DCHECK(dt_ctx->fmt_len > 0);
@@ -200,7 +190,7 @@ bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
dt_ctx->has_time_toks |= tok_type >= HOUR_IN_DAY;
if (!accept_time_toks && dt_ctx->has_time_toks) return false;
- // Get the token group length
+ // Get the token length
int tok_len = 1;
char tok_chr = *str;
const char* curr_tok_chr = str + 1;
@@ -214,8 +204,8 @@ bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
if (tok_len == 3) tok_type = MONTH_IN_YEAR_SLT;
}
// In an output scenario, fmt_out_len is used to determine the print buffer size.
- // If the format uses short token groups e.g. yyyy-MM-d, there must to be enough
- // room in the buffer for wider values e.g. 2013-12-16.
+ // If the format uses short tokens e.g. yyyy-MM-d, there must to be enough room in
+ // the buffer for wider values e.g. 2013-12-16.
if (tok_len == 1) ++dt_ctx->fmt_out_len;
DateTimeFormatToken tok(tok_type, str - str_begin, tok_len, str);
str += tok.len;
@@ -224,12 +214,8 @@ bool ParseFormatTokens(DateTimeFormatContext* dt_ctx, bool accept_time_toks) {
return dt_ctx->has_date_toks || dt_ctx->has_time_toks;
}
-// Parse out the next digit token from the date/time string by checking for contiguous
-// digit characters and return a pointer to the end of that token.
-// str -- pointer to the string to be parsed
-// str_end -- the pointer to the end of the string to be parsed
-// Returns the pointer within the string to the end of the valid digit token.
-const char* ParseDigitToken(const char* str, const char* str_end) {
+const char* SimpleDateFormatTokenizer::ParseDigitToken(const char* str,
+ const char* str_end) {
const char* tok_end = str;
while (tok_end < str_end) {
if (!isdigit(*tok_end)) return tok_end;
@@ -238,13 +224,8 @@ const char* ParseDigitToken(const char* str, const char* str_end) {
return tok_end;
}
-// Parse out the next separator token from the date/time string against an expected
-// character.
-// str -- pointer to the string to be parsed
-// str_end -- the pointer to the end of the string to be parsed
-// sep -- the separator char to compare the token to
-// Returns the pointer within the string to the end of the valid separator token.
-const char* ParseSeparatorToken(const char* str, const char* str_end, const char sep) {
+const char* SimpleDateFormatTokenizer::ParseSeparatorToken(const char* str,
+ const char* str_end, const char sep) {
const char* tok_end = str;
while (tok_end < str_end) {
if (*tok_end != sep) return tok_end;
@@ -253,8 +234,8 @@ const char* ParseSeparatorToken(const char* str, const char* str_end, const char
return tok_end;
}
-bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks,
- bool accept_time_toks_only) {
+bool SimpleDateFormatTokenizer::TokenizeByStr( DateTimeFormatContext* dt_ctx,
+ bool accept_time_toks, bool accept_time_toks_only) {
DCHECK(dt_ctx != NULL);
DCHECK(dt_ctx->fmt != NULL);
DCHECK_GT(dt_ctx->fmt_len, 0);
@@ -377,6 +358,7 @@ bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks
// Like the non-lazy path, this will parse up to 9 fractional digits
tok_end = ParseDigitToken(str, str_end);
int num_digits = std::min<int>(9, tok_end - str);
+ if (num_digits == 0) return false;
dt_ctx->toks.push_back(
DateTimeFormatToken(FRACTION, str - str_begin, num_digits, str));
str = tok_end;
@@ -387,9 +369,9 @@ bool ParseFormatTokensByStr(DateTimeFormatContext* dt_ctx, bool accept_time_toks
return true;
}
-const DateTimeFormatContext* ParseDefaultFormatTokensByStr(const char* str, int len,
- bool accept_time_toks, bool accept_time_toks_only) {
- DCHECK(IsParseCtxInitialized());
+const DateTimeFormatContext* SimpleDateFormatTokenizer::GetDefaultFormatContext(
+ const char* str, int len, bool accept_time_toks, bool accept_time_toks_only) {
+ DCHECK(initialized);
DCHECK(str != nullptr);
DCHECK(len > 0);
@@ -466,8 +448,9 @@ const DateTimeFormatContext* ParseDefaultFormatTokensByStr(const char* str, int
return nullptr;
}
-bool ParseDateTime(const char* str, int str_len, const DateTimeFormatContext& dt_ctx,
- DateTimeParseResult* dt_result) {
+bool SimpleDateFormatParser::ParseDateTime(const char* str, int str_len,
+ const DateTimeFormatContext& dt_ctx, DateTimeParseResult* dt_result) {
+ DCHECK(initialized);
DCHECK(dt_ctx.fmt_len > 0);
DCHECK(dt_ctx.toks.size() > 0);
DCHECK(dt_result != NULL);
@@ -493,18 +476,14 @@ bool ParseDateTime(const char* str, int str_len, const DateTimeFormatContext& dt
}
switch (tok.type) {
case YEAR: {
- dt_result->year = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- if (UNLIKELY(dt_result->year < 0 || dt_result->year > 9999)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 0, 9999, &dt_result->year)) return false;
// Year in "Y" and "YY" format should be in the interval
// [current time - 80 years, current time + 20 years)
if (tok_len <= 2) dt_result->realign_year = true;
break;
}
case MONTH_IN_YEAR: {
- dt_result->month = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- if (UNLIKELY(dt_result->month < 1 || dt_result->month > 12)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 1, 12, &dt_result->month)) return false;
break;
}
case MONTH_IN_YEAR_SLT: {
@@ -518,38 +497,23 @@ bool ParseDateTime(const char* str, int str_len, const DateTimeFormatContext& dt
break;
}
case DAY_IN_MONTH: {
- dt_result->day = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- // TODO: Validate that the value of day is correct for the given month.
- if (UNLIKELY(dt_result->day < 1 || dt_result->day > 31)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 1, 31, &dt_result->day)) return false;
break;
}
case HOUR_IN_DAY: {
- dt_result->hour = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- if (UNLIKELY(dt_result->hour < 0 || dt_result->hour > 23)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 0, 23, &dt_result->hour)) return false;
break;
}
case MINUTE_IN_HOUR: {
- dt_result->minute = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- if (UNLIKELY(dt_result->minute < 0 || dt_result->minute > 59)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 0, 59, &dt_result->minute)) return false;
break;
}
case SECOND_IN_MINUTE: {
- dt_result->second = StringParser::StringToInt<int>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- if (UNLIKELY(dt_result->second < 0 || dt_result->second > 59)) return false;
+ if (!ParseAndValidate(tok_val, tok_len, 0, 59, &dt_result->second)) return false;
break;
}
case FRACTION: {
- dt_result->fraction =
- StringParser::StringToInt<int32_t>(tok_val, tok_len, &status);
- if (UNLIKELY(StringParser::PARSE_SUCCESS != status)) return false;
- // A user may specify a time of 04:30:22.1238, the parser will return 1238 for
- // the fractional portion. This does not represent the intended value of
- // 123800000, therefore the number must be scaled up.
- for (int i = tok_len; i < 9; ++i) dt_result->fraction *= 10;
+ if (!ParseFractionToken(tok_val, tok_len, dt_result)) return false;
break;
}
case TZ_OFFSET: {
diff --git a/be/src/runtime/datetime-simple-date-format-parser.h b/be/src/runtime/datetime-simple-date-format-parser.h
new file mode 100644
index 0000000..740b27c
--- /dev/null
+++ b/be/src/runtime/datetime-simple-date-format-parser.h
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "runtime/datetime-parser-common.h"
+
+#include <boost/unordered_map.hpp>
+
+#include "runtime/string-value.h"
+
+namespace impala {
+
+/// The functionality here covers SimpleDateFormat pattern handling in Impala.
+/// Adds support for dealing with custom date/time formats in Impala. The following
+/// date/time tokens are supported:
+/// y - Year
+/// M - Month
+/// d - Day
+/// H - Hour
+/// m - Minute
+/// s - second
+/// S - Fractional second
+///
+/// TimeZone offset formats (Must be at the end of format string):
+/// +/-hh:mm
+/// +/-hhmm
+/// +/-hh
+///
+///
+/// The token names and usage have been modeled after the SimpleDateFormat class used in
+/// Java, with only the above list of tokens being supported. All fields will consume
+/// variable length inputs when parsing an input string and must therefore use separators
+/// to specify the boundaries of the fields, with the exception of TimeZone values, which
+/// have to be of fixed width. Repeating tokens can be used to specify fields of exact
+/// width, e.g. in yy-MM both fields must be of exactly length two. When using fixed width
+/// fields values must be zero-padded and output values will be zero padded during
+/// formatting. There is one exception to this: a month field of length 3 will specify
+/// literal month names instead of zero padding, i.e., yyyy-MMM-dd will parse from and
+/// format to strings like 2013-Nov-21. When using fields of fixed width the separators
+/// can be omitted.
+///
+///
+/// Formatting character groups can appear in any order along with any separators
+/// except TimeZone offset.
+/// e.g.
+/// yyyy/MM/dd
+/// dd-MMM-yy
+/// (dd)(MM)(yyyy) HH:mm:ss
+/// yyyy-MM-dd HH:mm:ss+hh:mm
+/// ..etc..
+///
+/// The following features are not supported:
+/// Long literal months e.g. MMMM
+/// Nested strings e.g. "Year: " yyyy "Month: " mm "Day: " dd
+/// Lazy formatting
+namespace datetime_parse_util {
+
+class SimpleDateFormatTokenizer {
+public:
+ /// Constants to hold default format lengths.
+ static const int DEFAULT_DATE_FMT_LEN;
+ static const int DEFAULT_TIME_FMT_LEN;
+ static const int DEFAULT_TIME_FRAC_FMT_LEN;
+ static const int DEFAULT_SHORT_DATE_TIME_FMT_LEN;
+ static const int DEFAULT_DATE_TIME_FMT_LEN;
+
+ /// Parse the date/time format into tokens and place them in the context.
+ /// dt_ctx -- output date/time format context
+ /// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
+ /// rejected.
+ /// Return true if the parse was successful.
+ static bool Tokenize(DateTimeFormatContext* dt_ctx,
+ bool accept_time_toks = true);
+
+ /// Parse the date/time string to generate the DateTimeFormatToken required by
+ /// DateTimeFormatContext. Similar to Tokenize() this function will take the string
+ /// and length, then heuristically determine whether the value contains date tokens,
+ /// time tokens, or both. Unlike Tokenize(), it does not require the template format
+ /// string.
+ /// dt_ctx -- date/time format context (must contain valid tokens)
+ /// accept_time_toks -- if true, time tokens are accepted, otherwise time tokens are
+ /// rejected.
+ /// accept_time_toks_only -- if true, time tokens w/o date tokens are accepted.
+ /// Otherwise, they are rejected.
+ /// Return true if the date/time was successfully parsed.
+ static bool TokenizeByStr(DateTimeFormatContext* dt_ctx,
+ bool accept_time_toks = true, bool accept_time_toks_only = true);
+
+ /// Parse date/time string to find the corresponding default date/time format context.
+ /// The string must adhere to a default date/time format.
+ /// str -- valid pointer to the string to parse.
+ /// len -- length of the string to parse (must be > 0)
+ /// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
+ /// rejected.
+ /// accept_time_toks_only -- if true, time tokens without date tokens are accepted.
+ /// Otherwise, they are rejected.
+ /// Return the corresponding default format context if parsing succeeded, or nullptr
+ /// otherwise.
+ static const DateTimeFormatContext* GetDefaultFormatContext(const char* str, int len,
+ bool accept_time_toks, bool accept_time_toks_only);
+
+ /// Initialize the default format contexts. This *must* be called before using
+ /// GetDefaultFormatContext().
+ static void InitCtx();
+private:
+ /// Used to indicate if the state has been initialized.
+ static bool initialized;
+
+ /// Pseudo-constant default date/time contexts. Backwards compatibility is provided on
+ /// variable length fractional components by defining a format context for each expected
+ /// length (0 - 9). This logic will be refactored when the parser supports lazy tokens.
+ static DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
+ static DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
+ static DateTimeFormatContext DEFAULT_DATE_CTX;
+ static DateTimeFormatContext DEFAULT_TIME_CTX;
+ static DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
+ static DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
+ static DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
+
+ /// Checks if str_begin point to the beginning of a valid timezone offset.
+ static bool IsValidTZOffset(const char* str_begin, const char* str_end);
+
+ // Parse out the next digit token from the date/time string by checking for contiguous
+ // digit characters and return a pointer to the end of that token.
+ // str -- pointer to the string to be parsed
+ // str_end -- the pointer to the end of the string to be parsed
+ // Returns the pointer within the string to the end of the valid digit token.
+ static const char* ParseDigitToken(const char* str, const char* str_end);
+
+ // Parse out the next separator token from the date/time string against an expected
+ // character.
+ // str -- pointer to the string to be parsed
+ // str_end -- the pointer to the end of the string to be parsed
+ // sep -- the separator char to compare the token to
+ // Returns the pointer within the string to the end of the valid separator token.
+ static const char* ParseSeparatorToken(const char* str, const char* str_end,
+ const char sep);
+};
+
+class SimpleDateFormatParser {
+public:
+ /// Parse a date/time string. The data must adhere to the format, otherwise it will be
+ /// rejected i.e. no missing tokens.
+ /// Does only a basic validation on the parsed date/time values. The caller is
+ /// responsible for implementing rigid data validation and range-check.
+ /// str -- valid pointer to the string to parse
+ /// len -- length of the string to parse (must be > 0)
+ /// dt_ctx -- date/time format context (must contain valid tokens)
+ /// dt_result -- the struct where the results of the parsing will be placed
+ /// Return true if the date/time was successfully parsed.
+ static bool ParseDateTime(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx, DateTimeParseResult* dt_result);
+
+ // Initializes REV_MONTH_INDEX;
+ static void InitCtx();
+private:
+ /// Used to indicate if the state has been initialized.
+ static bool initialized;
+
+ /// Lazily initialized pseudo-constant hashmap for mapping month names to an index.
+ static boost::unordered_map<StringValue, int> REV_MONTH_INDEX;
+};
+
+}
+
+}
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index 1443ae1..b16fb58 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -69,9 +69,10 @@ RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx& frag
: query_state_(query_state),
fragment_ctx_(&fragment_ctx),
instance_ctx_(&instance_ctx),
- now_(new TimestampValue(TimestampValue::Parse(query_state->query_ctx().now_string))),
- utc_timestamp_(new TimestampValue(
- TimestampValue::Parse(query_state->query_ctx().utc_timestamp_string))),
+ now_(new TimestampValue(TimestampValue::ParseSimpleDateFormat(
+ query_state->query_ctx().now_string))),
+ utc_timestamp_(new TimestampValue(TimestampValue::ParseSimpleDateFormat(
+ query_state->query_ctx().utc_timestamp_string))),
local_time_zone_(&TimezoneDatabase::GetUtcTimezone()),
profile_(RuntimeProfile::Create(
obj_pool(), "Fragment " + PrintId(instance_ctx.fragment_instance_id))),
@@ -92,8 +93,9 @@ RuntimeState::RuntimeState(
fragment_ctx_(nullptr),
instance_ctx_(nullptr),
local_query_state_(query_state_),
- now_(new TimestampValue(TimestampValue::Parse(qctx.now_string))),
- utc_timestamp_(new TimestampValue(TimestampValue::Parse(qctx.utc_timestamp_string))),
+ now_(new TimestampValue(TimestampValue::ParseSimpleDateFormat(qctx.now_string))),
+ utc_timestamp_(new TimestampValue(TimestampValue::ParseSimpleDateFormat(
+ qctx.utc_timestamp_string))),
local_time_zone_(&TimezoneDatabase::GetUtcTimezone()),
profile_(RuntimeProfile::Create(obj_pool(), "<unnamed>")),
instance_buffer_reservation_(nullptr) {
diff --git a/be/src/runtime/timestamp-parse-util.cc b/be/src/runtime/timestamp-parse-util.cc
index 629b044..d46b586 100644
--- a/be/src/runtime/timestamp-parse-util.cc
+++ b/be/src/runtime/timestamp-parse-util.cc
@@ -17,11 +17,16 @@
#include "runtime/timestamp-parse-util.h"
+#include "common/names.h"
+#include "runtime/datetime-iso-sql-format-parser.h"
+#include "runtime/datetime-simple-date-format-parser.h"
+#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
#include "runtime/timestamp-value.h"
+#include "udf/udf-internal.h"
#include "util/string-parser.h"
-#include "common/names.h"
+#include "cctype"
using boost::gregorian::date;
using boost::gregorian::date_duration;
@@ -35,17 +40,18 @@ namespace impala {
using namespace datetime_parse_util;
-// Helper for TimestampParse::Parse to produce return value and set output parameters
-// when parsing fails. 'd' and 't' must be non-NULL.
+// Helper for parse functions to produce return value and set output parameters when
+// parsing fails. 'd' and 't' must be non-NULL.
static bool IndicateTimestampParseFailure(date* d, time_duration* t) {
+ DCHECK(d != nullptr);
+ DCHECK(t != nullptr);
*d = date();
*t = time_duration(not_a_date_time);
return false;
}
-bool TimestampParser::Parse(const char* str, int len, boost::gregorian::date* d,
- boost::posix_time::time_duration* t) {
- DCHECK(IsParseCtxInitialized());
+bool TimestampParser::ParseSimpleDateFormat(const char* str, int len,
+ boost::gregorian::date* d, boost::posix_time::time_duration* t) {
DCHECK(d != nullptr);
DCHECK(t != nullptr);
if (UNLIKELY(str == nullptr)) return IndicateTimestampParseFailure(d, t);
@@ -61,9 +67,10 @@ bool TimestampParser::Parse(const char* str, int len, boost::gregorian::date* d,
// Strip if there is a 'Z' suffix
if (trimmed_len > 0 && str[trimmed_len - 1] == 'Z') {
--trimmed_len;
- } else if (trimmed_len > DEFAULT_TIME_FMT_LEN && (str[4] == '-' || str[2] == ':')) {
+ } else if (trimmed_len > SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN &&
+ (str[4] == '-' || str[2] == ':')) {
// Strip timezone offset if it seems like a valid timestamp string.
- int curr_pos = DEFAULT_TIME_FMT_LEN;
+ int curr_pos = SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN;
// Timezone offset will be at least two bytes long, no need to check last
// two bytes.
while (curr_pos < trimmed_len - 2) {
@@ -77,18 +84,24 @@ bool TimestampParser::Parse(const char* str, int len, boost::gregorian::date* d,
if (UNLIKELY(trimmed_len <= 0)) return IndicateTimestampParseFailure(d, t);
// Determine the length of relevant input, if we're using one of the default formats.
- int default_fmt_len = min(trimmed_len, DEFAULT_DATE_TIME_FMT_LEN);
+ int default_fmt_len = min(trimmed_len,
+ SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_FMT_LEN);
// Determine the default formatting context that's required for parsing.
- const DateTimeFormatContext* dt_ctx = ParseDefaultFormatTokensByStr(str,
- default_fmt_len, true, true);
+ const DateTimeFormatContext* dt_ctx =
+ SimpleDateFormatTokenizer::GetDefaultFormatContext(str, default_fmt_len, true,
+ true);
- if (dt_ctx != nullptr) return Parse(str, default_fmt_len, *dt_ctx, d, t);
+ if (dt_ctx != nullptr) {
+ return ParseSimpleDateFormat(str, default_fmt_len, *dt_ctx, d, t);
+ }
// Generating context lazily as a fall back if default formats fail.
// ParseFormatTokenByStr() does not require a template format string.
DateTimeFormatContext lazy_ctx(str, trimmed_len);
- if (!ParseFormatTokensByStr(&lazy_ctx)) return IndicateTimestampParseFailure(d, t);
+ if (!SimpleDateFormatTokenizer::TokenizeByStr(&lazy_ctx)) {
+ return IndicateTimestampParseFailure(d, t);
+ }
dt_ctx = &lazy_ctx;
- return Parse(str, trimmed_len, *dt_ctx, d, t);
+ return ParseSimpleDateFormat(str, trimmed_len, *dt_ctx, d, t);
}
date TimestampParser::RealignYear(const DateTimeParseResult& dt_result,
@@ -117,28 +130,26 @@ date TimestampParser::RealignYear(const DateTimeParseResult& dt_result,
}
}
-bool TimestampParser::Parse(const char* str, int len, const DateTimeFormatContext& dt_ctx,
- date* d, time_duration* t) {
- DCHECK(IsParseCtxInitialized());
- DCHECK(dt_ctx.toks.size() > 0);
- DCHECK(d != NULL);
- DCHECK(t != NULL);
- DateTimeParseResult dt_result;
- int day_offset = 0;
- if (UNLIKELY(str == NULL || len <= 0 || !ParseDateTime(str, len, dt_ctx, &dt_result))) {
- return IndicateTimestampParseFailure(d, t);
+int TimestampParser::AdjustWithTimezone(time_duration* t,
+ const time_duration& tz_offset) {
+ *t -= tz_offset;
+ if (t->is_negative()) {
+ *t += hours(24);
+ return -1;
+ } else if (t->hours() >= 24) {
+ *t -= hours(24);
+ return 1;
}
+ return 0;
+}
+
+bool TimestampParser::PopulateParseResult(const DateTimeFormatContext& dt_ctx,
+ const DateTimeParseResult& dt_result, date* d, time_duration* t) {
+ int day_offset = 0;
if (dt_ctx.has_time_toks) {
*t = time_duration(dt_result.hour, dt_result.minute,
dt_result.second, dt_result.fraction);
- *t -= dt_result.tz_offset;
- if (t->is_negative()) {
- *t += hours(24);
- day_offset = -1;
- } else if (t->hours() >= 24) {
- *t -= hours(24);
- day_offset = 1;
- }
+ day_offset = AdjustWithTimezone(t, dt_result.tz_offset);
} else {
*t = time_duration(0, 0, 0, 0);
}
@@ -161,8 +172,8 @@ bool TimestampParser::Parse(const char* str, int len, const DateTimeFormatContex
}
} catch (boost::exception&) {
VLOG_ROW << "Invalid date: " << dt_result.year << "-" << dt_result.month << "-"
- << dt_result.day;
- return IndicateTimestampParseFailure(d, t);
+ << dt_result.day;
+ return false;
}
} else {
*d = date();
@@ -170,24 +181,59 @@ bool TimestampParser::Parse(const char* str, int len, const DateTimeFormatContex
return true;
}
-int TimestampParser::Format(const DateTimeFormatContext& dt_ctx,
- const boost::gregorian::date& d, const boost::posix_time::time_duration& t,
- int len, char* buff) {
- DCHECK(IsParseCtxInitialized());
+bool TimestampParser::ParseSimpleDateFormat(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx, date* d, time_duration* t) {
DCHECK(dt_ctx.toks.size() > 0);
- DCHECK(len > dt_ctx.fmt_out_len);
- DCHECK(buff != NULL);
- if (dt_ctx.has_date_toks && d.is_special()) return -1;
- if (dt_ctx.has_time_toks && t.is_special()) return -1;
- char* str = buff;
+ DCHECK(d != nullptr);
+ DCHECK(t != nullptr);
+ DateTimeParseResult dt_result;
+ if (UNLIKELY(str == nullptr || len <= 0 ||
+ !SimpleDateFormatParser::ParseDateTime(str, len, dt_ctx, &dt_result))) {
+ return IndicateTimestampParseFailure(d, t);
+ }
+ if (!PopulateParseResult(dt_ctx, dt_result, d, t)) {
+ return IndicateTimestampParseFailure(d, t);
+ }
+ return true;
+}
+
+bool TimestampParser::ParseIsoSqlFormat(const char* str, int len,
+ const DateTimeFormatContext& dt_ctx, date* d, time_duration* t) {
+ DCHECK(dt_ctx.toks.size() > 0);
+ DCHECK(d != nullptr);
+ DCHECK(t != nullptr);
+ if (UNLIKELY(str == nullptr || len <= 0)) return IndicateTimestampParseFailure(d, t);
+
+ DateTimeParseResult dt_result;
+ if (!IsoSqlFormatParser::ParseDateTime(str, len, dt_ctx, &dt_result)) {
+ return IndicateTimestampParseFailure(d, t);
+ }
+
+ if (!PopulateParseResult(dt_ctx, dt_result, d, t)) {
+ return IndicateTimestampParseFailure(d, t);
+ }
+ return true;
+}
+
+string TimestampParser::Format(const DateTimeFormatContext& dt_ctx, const date& d,
+ const time_duration& t) {
+ DCHECK(dt_ctx.toks.size() > 0);
+ if (dt_ctx.has_date_toks && d.is_special()) return "";
+ if (dt_ctx.has_time_toks && t.is_special()) return "";
+ string result;
+ result.reserve(dt_ctx.fmt_out_len);
for (const DateTimeFormatToken& tok: dt_ctx.toks) {
int32_t num_val = -1;
const char* str_val = NULL;
int str_val_len = 0;
switch (tok.type) {
- case YEAR: {
+ case YEAR:
+ case ROUND_YEAR: {
num_val = d.year();
- if (tok.len <= 3) num_val %= 100;
+ if (tok.len < 4) {
+ int adjust_factor = std::pow(10, tok.len);
+ num_val %= adjust_factor;
+ }
break;
}
case MONTH_IN_YEAR: num_val = d.month().as_number(); break;
@@ -197,15 +243,40 @@ int TimestampParser::Format(const DateTimeFormatContext& dt_ctx,
break;
}
case DAY_IN_MONTH: num_val = d.day(); break;
+ case DAY_IN_YEAR: {
+ num_val = GetDayInYear(d.year(), d.month(), d.day());
+ break;
+ }
case HOUR_IN_DAY: num_val = t.hours(); break;
+ case HOUR_IN_HALF_DAY: {
+ num_val = t.hours();
+ if (num_val == 0) num_val = 12;
+ if (num_val > 12) num_val -= 12;
+ break;
+ }
+ case MERIDIEM_INDICATOR: {
+ const MERIDIEM_INDICATOR_TEXT* indicator_txt = (tok.len == 2) ? &AM : &AM_LONG;
+ if (t.hours() >= 12) {
+ indicator_txt = (tok.len == 2) ? &PM : &PM_LONG;
+ }
+ str_val_len = tok.len;
+ str_val = (isupper(*tok.val)) ? indicator_txt->first : indicator_txt->second;
+ break;
+ }
case MINUTE_IN_HOUR: num_val = t.minutes(); break;
case SECOND_IN_MINUTE: num_val = t.seconds(); break;
+ case SECOND_IN_DAY: {
+ num_val = t.hours() * 3600 + t.minutes() * 60 + t.seconds();
+ break;
+ }
case FRACTION: {
num_val = t.fractional_seconds();
if (num_val > 0) for (int j = tok.len; j < 9; ++j) num_val /= 10;
break;
}
- case SEPARATOR: {
+ case SEPARATOR:
+ case ISO8601_TIME_INDICATOR:
+ case ISO8601_ZULU_INDICATOR: {
str_val = tok.val;
str_val_len = tok.len;
break;
@@ -216,15 +287,15 @@ int TimestampParser::Format(const DateTimeFormatContext& dt_ctx,
default: DCHECK(false) << "Unknown date/time format token";
}
if (num_val > -1) {
- str += sprintf(str, "%0*d", tok.len, num_val);
+ string tmp_str = std::to_string(num_val);
+ if (tmp_str.length() < tok.len) tmp_str.insert(0, tok.len - tmp_str.length(), '0');
+ result.append(tmp_str);
} else {
- memcpy(str, str_val, str_val_len);
- str += str_val_len;
+ DCHECK(str_val != nullptr && str_val_len > 0);
+ result.append(str_val, str_val_len);
}
}
- /// Terminate the string
- *str = '\0';
- return str - buff;
+ return result;
}
}
diff --git a/be/src/runtime/timestamp-parse-util.h b/be/src/runtime/timestamp-parse-util.h
index ca309c3..e9b00a4 100644
--- a/be/src/runtime/timestamp-parse-util.h
+++ b/be/src/runtime/timestamp-parse-util.h
@@ -20,7 +20,8 @@
#include <boost/date_time/gregorian/gregorian.hpp>
-#include "runtime/datetime-parse-util.h"
+#include "gutil/macros.h"
+#include "runtime/datetime-parser-common.h"
namespace boost {
namespace posix_time {
@@ -44,33 +45,37 @@ class TimestampParser {
/// d -- the date value where the results of the parsing will be placed
/// t -- the time value where the results of the parsing will be placed
/// Returns true if the date/time was successfully parsed.
- static bool Parse(const char* str, int len, boost::gregorian::date* d,
- boost::posix_time::time_duration* t);
+ static bool ParseSimpleDateFormat(const char* str, int len, boost::gregorian::date* d,
+ boost::posix_time::time_duration* t) WARN_UNUSED_RESULT;
- /// Parse a date/time string. The data must adhere to the format, otherwise it will be
- /// rejected i.e. no missing tokens. In the case of just a date, the time will be set
- /// to 00:00:00. In the case of just a time, the date will be set to invalid.
+ /// Parse a date/time string. The data must adhere to SimpleDateFormat, otherwise it
+ /// will be rejected i.e. no missing tokens. In the case of just a date, the time will
+ /// be set to 00:00:00. In the case of just a time, the date will be set to invalid.
/// str -- valid pointer to the string to parse
/// len -- length of the string to parse (must be > 0)
/// dt_ctx -- date/time format context (must contain valid tokens)
/// d -- the date value where the results of the parsing will be placed
/// t -- the time value where the results of the parsing will be placed
/// Returns true if the date/time was successfully parsed.
- static bool Parse(const char* str, int len,
+ static bool ParseSimpleDateFormat(const char* str, int len,
const datetime_parse_util::DateTimeFormatContext& dt_ctx, boost::gregorian::date* d,
- boost::posix_time::time_duration* t);
+ boost::posix_time::time_duration* t) WARN_UNUSED_RESULT;
- /// Format the date/time values using the given format context. Note that a string
- /// terminator will be appended to the string.
+ /// Parse 'str' into date and time objects following ISO:SQL:2016 datetime pattern
+ /// format. This function is used when a user specifies a datetime format string for
+ /// the cast. 'dt_ctx' holds the format tokens produced by the tokenizer.
+ /// Returns true if the parsing succeeded. Otherwise it sets 'd' and 't' to
+ /// invalid values and returns false.
+ static bool ParseIsoSqlFormat(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx, boost::gregorian::date* d,
+ boost::posix_time::time_duration* t) WARN_UNUSED_RESULT;
+
+ /// Format the date/time values using the given format context.
/// dt_ctx -- date/time format context
/// d -- the date value
/// t -- the time value
- /// len -- the output buffer length (should be at least dt_ctx.fmt_exp_len + 1)
- /// buff -- the output string buffer (must be large enough to hold value)
- /// Return the number of characters copied in to the buffer (excluding terminator).
- static int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx,
- const boost::gregorian::date& d, const boost::posix_time::time_duration& t, int len,
- char* buff);
+ static std::string Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx,
+ const boost::gregorian::date& d, const boost::posix_time::time_duration& t);
private:
/// Helper function finding the correct century for 1 or 2 digit year according to
@@ -83,6 +88,20 @@ class TimestampParser {
const datetime_parse_util::DateTimeParseResult& dt_result,
const datetime_parse_util::DateTimeFormatContext& dt_ctx, int day_offset,
const boost::posix_time::time_duration& t);
+
+ /// Populates 'd' and 't' based on the outcome of parsing the input string into
+ /// 'dt_result'. 'dt_ctx' is used for checking if date, time and timezone tokens were
+ /// provided.
+ static bool PopulateParseResult(
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx,
+ const datetime_parse_util::DateTimeParseResult& dt_result,
+ boost::gregorian::date* d, boost::posix_time::time_duration* t) WARN_UNUSED_RESULT;
+
+ /// Adjusts the time in 't' with 'tz_offset' and handles overflow cases. After adjusting
+ /// with 'tz_offset' returns -1 if 't' is negative, +1 if 't' is greater than or equals
+ /// to 24 or zero otherwise.
+ static int AdjustWithTimezone(boost::posix_time::time_duration* t,
+ const boost::posix_time::time_duration& tz_offset);
};
}
diff --git a/be/src/runtime/timestamp-test.cc b/be/src/runtime/timestamp-test.cc
index ff4e3bb..535eb5e 100644
--- a/be/src/runtime/timestamp-test.cc
+++ b/be/src/runtime/timestamp-test.cc
@@ -23,7 +23,7 @@
#include "common/status.h"
#include "exprs/timezone_db.h"
-#include "runtime/datetime-parse-util.h"
+#include "runtime/datetime-simple-date-format-parser.h"
#include "runtime/raw-value.inline.h"
#include "runtime/timestamp-value.h"
#include "runtime/timestamp-value.inline.h"
@@ -41,8 +41,7 @@ using boost::posix_time::time_duration;
namespace impala {
-using datetime_parse_util::ParseFormatTokens;
-using datetime_parse_util::DateTimeFormatContext;
+using namespace datetime_parse_util;
// Used for defining a custom date/time format test. The structure can be used to
// indicate whether the format or value is expected to fail. In a happy path test,
@@ -185,17 +184,15 @@ void TestTimestampTokens(vector<TimestampToken>* toks, int year, int month,
}
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(ParseFormatTokens(&dt_ctx)) << fmt_val;
- TimestampValue tv = TimestampValue::Parse(val.c_str(), val.length(), dt_ctx);
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << fmt_val;
+ TimestampValue tv =
+ TimestampValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateTimestamp(tv, fmt, val, fmt_val, year, month, day, hours, mins, secs,
frac);
- int buff_len = dt_ctx.fmt_out_len + 1;
- char buff[buff_len];
- int actual_len = tv.Format(dt_ctx, buff_len, buff);
- EXPECT_GT(actual_len, 0) << fmt_val;
- EXPECT_LE(actual_len, dt_ctx.fmt_len) << fmt_val;
- string buff_str(buff);
- EXPECT_EQ(buff_str, val) << fmt_val << " " << buff_str;
+ string buff = tv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << fmt_val;
+ EXPECT_LE(buff.length(), dt_ctx.fmt_len) << fmt_val;
+ EXPECT_EQ(buff, val) << fmt_val << " " << buff;
fmt.clear();
val.clear();
}
@@ -215,17 +212,15 @@ void TestTimestampTokens(vector<TimestampToken>* toks, int year, int month,
}
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(ParseFormatTokens(&dt_ctx)) << fmt_val;
- TimestampValue tv = TimestampValue::Parse(val.c_str(), val.length(), dt_ctx);
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << fmt_val;
+ TimestampValue tv =
+ TimestampValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateTimestamp(tv, fmt, val, fmt_val, year, month, day, hours, mins, secs,
frac);
- int buff_len = dt_ctx.fmt_out_len + 1;
- char buff[buff_len];
- int actual_len = tv.Format(dt_ctx, buff_len, buff);
- EXPECT_GT(actual_len, 0) << fmt_val;
- EXPECT_LE(actual_len, dt_ctx.fmt_len) << fmt_val;
- string buff_str(buff);
- EXPECT_EQ(buff_str, val) << fmt_val << " " << buff_str;
+ string buff = tv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << fmt_val;
+ EXPECT_LE(buff.length(), dt_ctx.fmt_len) << fmt_val;
+ EXPECT_EQ(buff, val) << fmt_val << " " << buff;
fmt.clear();
val.clear();
}
@@ -342,9 +337,9 @@ TEST(TimestampTest, Basic) {
char s2[] = "1990-10-20 10:10:10.123456789 ";
char s3[] = " 1990-10-20 10:10:10.123456789";
- TimestampValue v1 = TimestampValue::Parse(s1, strlen(s1));
- TimestampValue v2 = TimestampValue::Parse(s2, strlen(s2));
- TimestampValue v3 = TimestampValue::Parse(s3, strlen(s3));
+ TimestampValue v1 = TimestampValue::ParseSimpleDateFormat(s1, strlen(s1));
+ TimestampValue v2 = TimestampValue::ParseSimpleDateFormat(s2, strlen(s2));
+ TimestampValue v3 = TimestampValue::ParseSimpleDateFormat(s3, strlen(s3));
EXPECT_EQ(v1.date().year(), 2012);
EXPECT_EQ(v1.date().month(), 1);
@@ -370,8 +365,8 @@ TEST(TimestampTest, Basic) {
char s4[] = "2012-01-20T01:10:01";
char s5[] = "1990-10-20T10:10:10.123456789";
- TimestampValue v4 = TimestampValue::Parse(s4, strlen(s4));
- TimestampValue v5 = TimestampValue::Parse(s5, strlen(s5));
+ TimestampValue v4 = TimestampValue::ParseSimpleDateFormat(s4, strlen(s4));
+ TimestampValue v5 = TimestampValue::ParseSimpleDateFormat(s5, strlen(s5));
EXPECT_EQ(v4.date().year(), 2012);
EXPECT_EQ(v4.date().month(), 1);
@@ -391,8 +386,8 @@ TEST(TimestampTest, Basic) {
// Test Dates and Times as timestamps.
char d1[] = "2012-01-20";
char d2[] = "1990-10-20";
- TimestampValue dv1 = TimestampValue::Parse(d1, strlen(d1));
- TimestampValue dv2 = TimestampValue::Parse(d2, strlen(d2));
+ TimestampValue dv1 = TimestampValue::ParseSimpleDateFormat(d1, strlen(d1));
+ TimestampValue dv2 = TimestampValue::ParseSimpleDateFormat(d2, strlen(d2));
EXPECT_NE(dv1, dv2);
EXPECT_LT(dv1, v1);
@@ -407,8 +402,8 @@ TEST(TimestampTest, Basic) {
char t1[] = "10:11:12.123456789";
char t2[] = "00:00:00";
- TimestampValue tv1 = TimestampValue::Parse(t1, strlen(t1));
- TimestampValue tv2 = TimestampValue::Parse(t2, strlen(t2));
+ TimestampValue tv1 = TimestampValue::ParseSimpleDateFormat(t1, strlen(t1));
+ TimestampValue tv2 = TimestampValue::ParseSimpleDateFormat(t2, strlen(t2));
EXPECT_NE(tv1, tv2);
EXPECT_NE(tv1, v2);
@@ -432,7 +427,8 @@ TEST(TimestampTest, Basic) {
memcpy(frac_buff, TEST_VALS[i], VAL_LEN);
memcpy(frac_buff + VAL_LEN, FRACTION_MAX_STR, fraction_len);
*(frac_buff + VAL_LEN + fraction_len) = '\0';
- TimestampValue tv_frac = TimestampValue::Parse(frac_buff, strlen(frac_buff));
+ TimestampValue tv_frac =
+ TimestampValue::ParseSimpleDateFormat(frac_buff, strlen(frac_buff));
if (frac_buff[4] == '-') {
EXPECT_EQ(tv_frac.date().year(), 2013);
EXPECT_EQ(tv_frac.date().month(), 12);
@@ -453,44 +449,44 @@ TEST(TimestampTest, Basic) {
// Bad formats
char b1[] = "1990-10 10:10:10.123456789";
- TimestampValue bv1 = TimestampValue::Parse(b1, strlen(b1));
+ TimestampValue bv1 = TimestampValue::ParseSimpleDateFormat(b1, strlen(b1));
boost::gregorian::date not_a_date;
EXPECT_EQ(bv1.date(), not_a_date);
EXPECT_EQ(bv1.time(), not_a_date_time);
char b2[] = "1991-10-10 99:10:10.123456789";
- TimestampValue bv2 = TimestampValue::Parse(b2, strlen(b2));
+ TimestampValue bv2 = TimestampValue::ParseSimpleDateFormat(b2, strlen(b2));
EXPECT_EQ(bv2.time(), not_a_date_time);
EXPECT_EQ(bv2.date(), not_a_date);
char b3[] = "1990-10- 10:10:10.123456789";
- TimestampValue bv3 = TimestampValue::Parse(b3, strlen(b3));
+ TimestampValue bv3 = TimestampValue::ParseSimpleDateFormat(b3, strlen(b3));
EXPECT_EQ(bv3.date(), not_a_date);
EXPECT_EQ(bv3.time(), not_a_date_time);
char b4[] = "10:1010.123456789";
- TimestampValue bv4 = TimestampValue::Parse(b4, strlen(b4));
+ TimestampValue bv4 = TimestampValue::ParseSimpleDateFormat(b4, strlen(b4));
EXPECT_EQ(bv4.date(), not_a_date);
EXPECT_EQ(bv4.time(), not_a_date_time);
char b5[] = "10:11:12.123456 1991-10-10";
- TimestampValue bv5 = TimestampValue::Parse(b5, strlen(b5));
+ TimestampValue bv5 = TimestampValue::ParseSimpleDateFormat(b5, strlen(b5));
EXPECT_EQ(bv5.date(), not_a_date);
EXPECT_EQ(bv5.time(), not_a_date_time);
char b6[] = "2012-01-20 01:10:00.123.466";
- TimestampValue bv6 = TimestampValue::Parse(b6, strlen(b6));
+ TimestampValue bv6 = TimestampValue::ParseSimpleDateFormat(b6, strlen(b6));
EXPECT_EQ(bv6.date(), not_a_date);
EXPECT_EQ(bv6.time(), not_a_date_time);
char b7[] = "2012-01-20 01:10:00.123 477 ";
- TimestampValue bv7 = TimestampValue::Parse(b7, strlen(b7));
+ TimestampValue bv7 = TimestampValue::ParseSimpleDateFormat(b7, strlen(b7));
EXPECT_EQ(bv7.date(), not_a_date);
EXPECT_EQ(bv7.time(), not_a_date_time);
@@ -643,15 +639,16 @@ TEST(TimestampTest, Basic) {
for (int i = 0; i < test_cases.size(); ++i) {
TimestampTC test_case = test_cases[i];
DateTimeFormatContext dt_ctx(test_case.fmt);
- dt_ctx.SetCenturyBreak(now);
- bool parse_result = ParseFormatTokens(&dt_ctx);
+ dt_ctx.SetCenturyBreakAndCurrentTime(now);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx);
if (test_case.fmt_should_fail) {
EXPECT_FALSE(parse_result) << "TC: " << i;
continue;
} else {
ASSERT_TRUE(parse_result) << "TC: " << i;
}
- TimestampValue cust_tv = TimestampValue::Parse(test_case.str, strlen(test_case.str), dt_ctx);
+ TimestampValue cust_tv = TimestampValue::ParseSimpleDateFormat(test_case.str,
+ strlen(test_case.str), dt_ctx);
boost::gregorian::date cust_date = cust_tv.date();
boost::posix_time::time_duration cust_time = cust_tv.time();
if (test_case.str_should_fail) {
@@ -681,13 +678,10 @@ TEST(TimestampTest, Basic) {
EXPECT_EQ(test_case.expected_fraction, cust_time.fractional_seconds()) << "TC: "
<< i;
if (!test_case.should_format) continue;
- int buff_len = dt_ctx.fmt_out_len + 1;
- char buff[buff_len];
- int actual_len = cust_tv.Format(dt_ctx, buff_len, buff);
- EXPECT_GT(actual_len, 0) << "TC: " << i;
- EXPECT_LE(actual_len, dt_ctx.fmt_len) << "TC: " << i;
- EXPECT_EQ(string(test_case.str, strlen(test_case.str)), string(buff, actual_len))
- << "TC: " << i;
+ string buff = cust_tv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << "TC: " << i;
+ EXPECT_LE(buff.length(), dt_ctx.fmt_len) << "TC: " << i;
+ EXPECT_EQ(string(test_case.str, strlen(test_case.str)), buff) << "TC: " << i;
} else {
EXPECT_EQ(cust_time, not_a_date_time) << test_case.fmt << " " << test_case.str;
}
@@ -718,25 +712,22 @@ TEST(TimestampTest, Basic) {
for (int i = 0; i < fmt_test_cases.size(); ++i) {
TimestampFormatTC test_case = fmt_test_cases[i];
DateTimeFormatContext dt_ctx(test_case.fmt);
- ASSERT_TRUE(ParseFormatTokens(&dt_ctx)) << "TC: " << i;
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << "TC: " << i;
TimestampValue cust_tv = TimestampValue::FromUnixTime(test_case.ts, utc_tz);
EXPECT_NE(cust_tv.date(), not_a_date) << "TC: " << i;
EXPECT_NE(cust_tv.time(), not_a_date_time) << "TC: " << i;
EXPECT_GE(dt_ctx.fmt_out_len, dt_ctx.fmt_len);
- int buff_len = dt_ctx.fmt_out_len + 1;
- char buff[buff_len];
- int actual_len = cust_tv.Format(dt_ctx, buff_len, buff);
- EXPECT_GT(actual_len, 0) << "TC: " << i;
- EXPECT_LE(actual_len, dt_ctx.fmt_out_len) << "TC: " << i;
- EXPECT_EQ(string(buff, actual_len),
- string(test_case.str, strlen(test_case.str))) << "TC: " << i;
+ string buff = cust_tv.Format(dt_ctx);
+ EXPECT_TRUE(!buff.empty()) << "TC: " << i;
+ EXPECT_LE(buff.length(), dt_ctx.fmt_out_len) << "TC: " << i;
+ EXPECT_EQ(buff, string(test_case.str, strlen(test_case.str))) << "TC: " << i;
}
// Test rounding near edge cases.
const int64_t MIN_DATE_AS_UNIX_TIME = -17987443200;
{
// Check lowest valid timestamp.
- const TimestampValue ts = TimestampValue::Parse("1400-01-01");
+ const TimestampValue ts = TimestampValue::ParseSimpleDateFormat("1400-01-01");
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, RoundToMicros(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
@@ -745,7 +736,8 @@ TEST(TimestampTest, Basic) {
{
// Check that 250 nanoseconds is rounded/floored to last microsecond.
- const TimestampValue ts = TimestampValue::Parse("1400-01-01 00:00:00.000000250");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00.000000250");
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, RoundToMicros(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
@@ -754,7 +746,8 @@ TEST(TimestampTest, Basic) {
{
// Check that 500 nanosecond is rounded up to the next microsecond, while floored to
// the last microsecond.
- const TimestampValue ts = TimestampValue::Parse("1400-01-01 00:00:00.000000500");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00.000000500");
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC + 1, RoundToMicros(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
@@ -762,14 +755,16 @@ TEST(TimestampTest, Basic) {
{
// Check that 250 microseconds is floored to last millisecond.
- const TimestampValue ts = TimestampValue::Parse("1400-01-01 00:00:00.000250");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00.000250");
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MILLIS_PER_SEC, FloorToMillis(ts));
}
{
// Check that 500 microseconds is floored to last millisecond.
- const TimestampValue ts = TimestampValue::Parse("1400-01-01 00:00:00.000500");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00.000500");
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MIN_DATE_AS_UNIX_TIME * MILLIS_PER_SEC, FloorToMillis(ts));
}
@@ -795,7 +790,8 @@ TEST(TimestampTest, Basic) {
const int64_t MAX_DATE_AS_UNIX_TIME = 253402300799;
{
// Test the max supported date that can be represented in seconds.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59");
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC, RoundToMicros(ts));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
@@ -804,7 +800,8 @@ TEST(TimestampTest, Basic) {
{
// Check that 250 nanoseconds is rounded/floored to last microsecond.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59.000000250");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.000000250");
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC, RoundToMicros(ts));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
}
@@ -812,21 +809,24 @@ TEST(TimestampTest, Basic) {
{
// Check that 500 nanosecond is rounded to the next microsecond, while floored to the
// last microsecond.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59.000000500");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.000000500");
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC + 1, RoundToMicros(ts));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC, FloorToMicros(ts));
}
{
// Check that 250 microseconds is floored to last millisecond.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59.000250");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.000250");
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME, FloorToSeconds(ts));
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MILLIS_PER_SEC, FloorToMillis(ts));
}
{
// Check that 500 microseconds is floored to last millisecond.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59.000500");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.000500");
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MILLIS_PER_SEC, FloorToMillis(ts));
}
@@ -834,7 +834,8 @@ TEST(TimestampTest, Basic) {
// The max date that can be represented with the maximum number of nanoseconds. Unlike
// the cases above, rounding to microsecond does not round up to the
// next microsecond because that time is not supported by Impala.
- const TimestampValue ts = TimestampValue::Parse("9999-12-31 23:59:59.999999999");
+ const TimestampValue ts =
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59.999999999");
// The result is the maximum date with the maximum number of microseconds supported by
// Impala.
EXPECT_EQ(MAX_DATE_AS_UNIX_TIME * MICROS_PER_SEC + 999999, RoundToMicros(ts));
@@ -920,7 +921,8 @@ TEST(TimestampTest, Basic) {
EXPECT_EQ("1999-01-01 00:00:00",
TimestampValue::FromUnixTime(915148800, utc_tz).ToString());
// The leap second doesn't parse in Impala.
- TimestampValue leap_tv = TimestampValue::Parse("1998-12-31 23:59:60.00");
+ TimestampValue leap_tv =
+ TimestampValue::ParseSimpleDateFormat("1998-12-31 23:59:60.00");
EXPECT_FALSE(leap_tv.HasDateAndTime());
// The leap second can be parsed by ptime, though it is just converted to the time
@@ -943,7 +945,7 @@ TEST(TimestampTest, Basic) {
// Test Unix time as a float
double result;
EXPECT_TRUE(
- TimestampValue::Parse("2013-10-21 06:43:12.07").ToSubsecondUnixTime(
+ TimestampValue::ParseSimpleDateFormat("2013-10-21 06:43:12.07").ToSubsecondUnixTime(
utc_tz, &result));
EXPECT_EQ(1382337792.07, result);
EXPECT_EQ("1970-01-01 00:00:00.008000000",
@@ -1000,7 +1002,7 @@ TEST(TimestampTest, SubSecond) {
// Convenience function to create TimestampValues from strings.
TimestampValue StrToTs(const char * str) {
- return TimestampValue::Parse(str);
+ return TimestampValue::ParseSimpleDateFormat(str);
}
TEST(TimestampTest, TimezoneConversions) {
diff --git a/be/src/runtime/timestamp-value.cc b/be/src/runtime/timestamp-value.cc
index 74fa6ba..01d3238 100644
--- a/be/src/runtime/timestamp-value.cc
+++ b/be/src/runtime/timestamp-value.cc
@@ -53,26 +53,34 @@ using datetime_parse_util::DateTimeFormatContext;
const char* TimestampValue::LLVM_CLASS_NAME = "class.impala::TimestampValue";
const double TimestampValue::ONE_BILLIONTH = 0.000000001;
-TimestampValue TimestampValue::Parse(const char* str, int len) {
+TimestampValue TimestampValue::ParseSimpleDateFormat(const char* str, int len) {
TimestampValue tv;
- TimestampParser::Parse(str, len, &tv.date_, &tv.time_);
+ discard_result(TimestampParser::ParseSimpleDateFormat(str, len, &tv.date_, &tv.time_));
return tv;
}
-TimestampValue TimestampValue::Parse(const string& str) {
- return Parse(str.c_str(), str.size());
+TimestampValue TimestampValue::ParseSimpleDateFormat(const string& str) {
+ return ParseSimpleDateFormat(str.c_str(), str.size());
}
-TimestampValue TimestampValue::Parse(const char* str, int len,
+TimestampValue TimestampValue::ParseSimpleDateFormat(const char* str, int len,
const DateTimeFormatContext& dt_ctx) {
TimestampValue tv;
- TimestampParser::Parse(str, len, dt_ctx, &tv.date_, &tv.time_);
+ discard_result(TimestampParser::ParseSimpleDateFormat(str, len, dt_ctx, &tv.date_,
+ &tv.time_));
return tv;
}
-int TimestampValue::Format(const DateTimeFormatContext& dt_ctx, int len, char* buff)
- const {
- return TimestampParser::Format(dt_ctx, date_, time_, len, buff);
+TimestampValue TimestampValue::ParseIsoSqlFormat(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx) {
+ TimestampValue tv;
+ discard_result(TimestampParser::ParseIsoSqlFormat(str, len, dt_ctx, &tv.date_,
+ &tv.time_));
+ return tv;
+}
+
+string TimestampValue::Format(const DateTimeFormatContext& dt_ctx) const {
+ return TimestampParser::Format(dt_ctx, date_, time_);
}
namespace {
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index bb503e5..2b2ce79 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -88,9 +88,11 @@ class TimestampValue {
/// Constructors that parse from a date/time string. See TimestampParser for details
/// about the date-time format.
- static TimestampValue Parse(const std::string& str);
- static TimestampValue Parse(const char* str, int len);
- static TimestampValue Parse(const char* str, int len,
+ static TimestampValue ParseSimpleDateFormat(const std::string& str);
+ static TimestampValue ParseSimpleDateFormat(const char* str, int len);
+ static TimestampValue ParseSimpleDateFormat(const char* str, int len,
+ const datetime_parse_util::DateTimeFormatContext& dt_ctx);
+ static TimestampValue ParseIsoSqlFormat(const char* str, int len,
const datetime_parse_util::DateTimeFormatContext& dt_ctx);
/// 'days' represents the number of days since 1970-01-01.
@@ -206,15 +208,9 @@ class TimestampValue {
&& time.total_nanoseconds() < NANOS_PER_DAY;
}
- /// Formats the timestamp using the given date/time context and places the result in the
- /// string buffer. The size of the buffer should be at least dt_ctx.fmt_out_len + 1. A
- /// string terminator will be appended to the string.
+ /// Formats the timestamp using the given date/time context and returns the result.
/// dt_ctx -- the date/time context containing the format to use
- /// len -- the length of the buffer
- /// buff -- the buffer that will hold the result
- /// Returns the number of characters copied in to the buffer (minus the terminator)
- int Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx, int len,
- char* buff) const;
+ std::string Format(const datetime_parse_util::DateTimeFormatContext& dt_ctx) const;
/// Interpret 'this' as a timestamp in UTC and convert to unix time.
/// Returns false if the conversion failed ('unix_time' will be undefined), otherwise
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index b6abb0a..73a9495 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -988,7 +988,12 @@ void ImpalaServer::PrepareQueryContext(const TNetworkAddress& backend_addr,
local_tz = &utc_tz;
}
query_ctx->__set_utc_timestamp_string(ToStringFromUnixMicros(now_us, utc_tz));
- query_ctx->__set_now_string(ToStringFromUnixMicros(now_us, *local_tz));
+ if (query_ctx->client_request.query_options.now_string.empty()) {
+ query_ctx->__set_now_string(ToStringFromUnixMicros(now_us, *local_tz));
+ } else {
+ // For testing purposes
+ query_ctx->__set_now_string(query_ctx->client_request.query_options.now_string);
+ }
query_ctx->__set_start_unix_millis(now_us / MICROS_PER_MILLI);
query_ctx->__set_coord_address(backend_addr);
query_ctx->__set_coord_krpc_address(krpc_addr);
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index 1b684d3..ea89eb5 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -877,6 +877,10 @@ Status impala::SetQueryOption(const string& key, const string& value,
query_options->__set_fetch_rows_timeout_ms(requested_timeout);
break;
}
+ case TImpalaQueryOptions::NOW_STRING: {
+ query_options->__set_now_string(value);
+ break;
+ }
default:
if (IsRemovedQueryOption(key)) {
LOG(WARNING) << "Ignoring attempt to set removed query option '" << key << "'";
diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h
index 273bcff..843afc9 100644
--- a/be/src/service/query-options.h
+++ b/be/src/service/query-options.h
@@ -47,7 +47,7 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
// time we add or remove a query option to/from the enum TImpalaQueryOptions.
#define QUERY_OPTS_TABLE\
DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(),\
- TImpalaQueryOptions::FETCH_ROWS_TIMEOUT_MS + 1);\
+ TImpalaQueryOptions::NOW_STRING + 1);\
REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded, ABORT_ON_DEFAULT_LIMIT_EXCEEDED)\
QUERY_OPT_FN(abort_on_error, ABORT_ON_ERROR, TQueryOptionLevel::REGULAR)\
REMOVED_QUERY_OPT_FN(allow_unsupported_formats, ALLOW_UNSUPPORTED_FORMATS)\
@@ -188,7 +188,8 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
QUERY_OPT_FN(disable_hbase_num_rows_estimate, DISABLE_HBASE_NUM_ROWS_ESTIMATE,\
TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(fetch_rows_timeout_ms, FETCH_ROWS_TIMEOUT_MS,\
- TQueryOptionLevel::ADVANCED)
+ TQueryOptionLevel::ADVANCED)\
+ QUERY_OPT_FN(now_string, NOW_STRING, TQueryOptionLevel::DEVELOPMENT)\
;
/// Enforce practical limits on some query options to avoid undesired query state.
diff --git a/be/src/testutil/random-vector-generators.h b/be/src/testutil/random-vector-generators.h
index b26820e..780e9b1 100644
--- a/be/src/testutil/random-vector-generators.h
+++ b/be/src/testutil/random-vector-generators.h
@@ -85,13 +85,13 @@ std::vector<std::string> RandomStrVec(Generator& gen, const int length,
template <typename Generator = std::mt19937>
std::vector<TimestampValue> RandomTimestampVec(Generator& gen, const int length) {
const TimestampValue min_date =
- TimestampValue::Parse("1400-01-01 00:00:00");
+ TimestampValue::ParseSimpleDateFormat("1400-01-01 00:00:00");
int64_t min_millis;
bool min_success = min_date.FloorUtcToUnixTimeMillis(&min_millis);
DCHECK(min_success);
const TimestampValue max_date =
- TimestampValue::Parse("9999-12-31 23:59:59");
+ TimestampValue::ParseSimpleDateFormat("9999-12-31 23:59:59");
int64_t max_millis;
bool max_success = max_date.FloorUtcToUnixTimeMillis(&max_millis);
DCHECK(max_success);
diff --git a/be/src/util/dict-test.cc b/be/src/util/dict-test.cc
index 875d4c3..73c64a5 100644
--- a/be/src/util/dict-test.cc
+++ b/be/src/util/dict-test.cc
@@ -129,9 +129,9 @@ TEST(DictTest, TestStrings) {
}
TEST(DictTest, TestTimestamps) {
- TimestampValue tv1 = TimestampValue::Parse("2011-01-01 09:01:01");
- TimestampValue tv2 = TimestampValue::Parse("2012-01-01 09:01:01");
- TimestampValue tv3 = TimestampValue::Parse("2011-01-01 09:01:02");
+ TimestampValue tv1 = TimestampValue::ParseSimpleDateFormat("2011-01-01 09:01:01");
+ TimestampValue tv2 = TimestampValue::ParseSimpleDateFormat("2012-01-01 09:01:01");
+ TimestampValue tv3 = TimestampValue::ParseSimpleDateFormat("2011-01-01 09:01:02");
vector<TimestampValue> dict_values;
dict_values.push_back(tv1);
diff --git a/be/src/util/min-max-filter-test.cc b/be/src/util/min-max-filter-test.cc
index 8ae20b7..a75ca1a 100644
--- a/be/src/util/min-max-filter-test.cc
+++ b/be/src/util/min-max-filter-test.cc
@@ -329,16 +329,16 @@ TEST(MinMaxFilterTest, TestTimestampMinMaxFilter) {
EXPECT_FALSE(empty_filter->AlwaysTrue());
// Now insert some stuff.
- TimestampValue t1 = TimestampValue::Parse("2000-01-01 00:00:00");
+ TimestampValue t1 = TimestampValue::ParseSimpleDateFormat("2000-01-01 00:00:00");
filter->Insert(&t1);
CheckTimestampVals(filter, t1, t1);
- TimestampValue t2 = TimestampValue::Parse("1990-01-01 12:30:00");
+ TimestampValue t2 = TimestampValue::ParseSimpleDateFormat("1990-01-01 12:30:00");
filter->Insert(&t2);
CheckTimestampVals(filter, t2, t1);
- TimestampValue t3 = TimestampValue::Parse("2001-04-30 05:00:00");
+ TimestampValue t3 = TimestampValue::ParseSimpleDateFormat("2001-04-30 05:00:00");
filter->Insert(&t3);
CheckTimestampVals(filter, t2, t3);
- TimestampValue t4 = TimestampValue::Parse("2001-04-30 01:00:00");
+ TimestampValue t4 = TimestampValue::ParseSimpleDateFormat("2001-04-30 01:00:00");
filter->Insert(&t4);
CheckTimestampVals(filter, t2, t3);
diff --git a/be/src/util/string-parser.h b/be/src/util/string-parser.h
index 51ed461..a688d24 100644
--- a/be/src/util/string-parser.h
+++ b/be/src/util/string-parser.h
@@ -103,14 +103,16 @@ class StringParser {
ParseResult* result) {
boost::gregorian::date d;
boost::posix_time::time_duration t;
- *result = TimestampParser::Parse(s, len, &d, &t) ? PARSE_SUCCESS : PARSE_FAILURE;
+ *result = TimestampParser::ParseSimpleDateFormat(s, len, &d, &t) ? PARSE_SUCCESS :
+ PARSE_FAILURE;
return {d, t};
}
/// Parse a DateValue from s.
static inline DateValue StringToDate(const char* s, int len, ParseResult* result) {
DateValue d;
- *result = DateParser::Parse(s, len, false, &d) ? PARSE_SUCCESS : PARSE_FAILURE;
+ *result = DateParser::ParseSimpleDateFormat(s, len, false, &d) ? PARSE_SUCCESS :
+ PARSE_FAILURE;
return d;
}
diff --git a/common/thrift/Exprs.thrift b/common/thrift/Exprs.thrift
index d496376..7ed771a 100644
--- a/common/thrift/Exprs.thrift
+++ b/common/thrift/Exprs.thrift
@@ -141,6 +141,11 @@ struct TKuduPartitionExpr {
2: required list<i32> referenced_columns
}
+struct TCastExpr {
+ // Holds the format clause of a cast expression.
+ 1: required string cast_format
+}
+
// This is essentially a union over the subclasses of Expr.
struct TExprNode {
1: required TExprNodeType node_type
@@ -171,6 +176,7 @@ struct TExprNode {
19: optional TAggregateExpr agg_expr
20: optional TTimestampLiteral timestamp_literal
21: optional TKuduPartitionExpr kudu_partition_expr
+ 22: optional TCastExpr cast_expr
}
// A flattened representation of a tree of Expr nodes, obtained by depth-first
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index cdc6a63..d64f59c 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -394,6 +394,9 @@ struct TQueryOptions {
// See comment in ImpalaService.thrift
94: optional i64 fetch_rows_timeout_ms = 10000;
+
+ // For testing purposes
+ 95: optional string now_string = "";
}
// Impala currently has two types of sessions: Beeswax and HiveServer2
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 8bfeace..fa6797c 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -460,6 +460,10 @@ enum TImpalaQueryOptions {
// client waits for all returned RowBatches to be produced. If the timeout is hit, the
// client returns whatever rows it has already read. Defaults to 10000 milliseconds.
FETCH_ROWS_TIMEOUT_MS = 93
+
+ // For testing purposes only. This can provide a datetime string to use as now() for
+ // tests.
+ NOW_STRING = 94
}
// The summary of a DML statement.
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index df6fbf1..e428b2d 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -386,6 +386,7 @@ nonterminal Expr opt_offset_param;
nonterminal LimitElement opt_limit_offset_clause;
nonterminal Expr opt_limit_clause, opt_offset_clause;
nonterminal Expr cast_expr, case_else_clause, analytic_expr;
+nonterminal String cast_format_val;
nonterminal Expr function_call_expr;
nonterminal AnalyticWindow opt_window_clause;
nonterminal AnalyticWindow.Type window_type;
@@ -2933,9 +2934,16 @@ opt_offset_clause ::=
{: RESULT = null; :}
;
+cast_format_val ::=
+ KW_FORMAT STRING_LITERAL:pattern
+ {: RESULT = pattern; :}
+ | /* empty */
+ {: RESULT = null; :}
+ ;
+
cast_expr ::=
- KW_CAST LPAREN expr:e KW_AS type_def:targetType RPAREN
- {: RESULT = new CastExpr(targetType, e); :}
+ KW_CAST LPAREN expr:e KW_AS type_def:targetType cast_format_val:format RPAREN
+ {: RESULT = new CastExpr(targetType, e, format); :}
;
case_expr ::=
diff --git a/fe/src/main/java/org/apache/impala/analysis/CastExpr.java b/fe/src/main/java/org/apache/impala/analysis/CastExpr.java
index 5fd853e..8b40451 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CastExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CastExpr.java
@@ -26,6 +26,7 @@ import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.ScalarType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TCastExpr;
import org.apache.impala.thrift.TExpr;
import org.apache.impala.thrift.TExprNode;
import org.apache.impala.thrift.TExprNodeType;
@@ -47,16 +48,20 @@ public class CastExpr extends Expr {
// Prefix for naming cast functions.
protected final static String CAST_FUNCTION_PREFIX = "castto";
+ // Stores the value of the FORMAT clause.
+ private final String castFormat_;
+
/**
* C'tor for "pre-analyzed" implicit casts.
*/
- public CastExpr(Type targetType, Expr e) {
+ public CastExpr(Type targetType, Expr e, String format) {
super();
Preconditions.checkState(targetType.isValid());
Preconditions.checkNotNull(e);
type_ = targetType;
targetTypeDef_ = null;
isImplicit_ = true;
+ castFormat_ = format;
// replace existing implicit casts
if (e instanceof CastExpr) {
CastExpr castExpr = (CastExpr) e;
@@ -78,15 +83,24 @@ public class CastExpr extends Expr {
analysisDone();
}
+ public CastExpr(Type targetType, Expr e) {
+ this(targetType, e, null);
+ }
+
/**
* C'tor for explicit casts.
*/
public CastExpr(TypeDef targetTypeDef, Expr e) {
+ this(targetTypeDef, e, null);
+ }
+
+ public CastExpr(TypeDef targetTypeDef, Expr e, String format) {
Preconditions.checkNotNull(targetTypeDef);
Preconditions.checkNotNull(e);
isImplicit_ = false;
targetTypeDef_ = targetTypeDef;
children_.add(e);
+ castFormat_ = format;
}
/**
@@ -97,6 +111,7 @@ public class CastExpr extends Expr {
targetTypeDef_ = other.targetTypeDef_;
isImplicit_ = other.isImplicit_;
noOp_ = other.noOp_;
+ castFormat_ = other.castFormat_;
}
private static String getFnName(Type targetType) {
@@ -183,14 +198,18 @@ public class CastExpr extends Expr {
public String toSqlImpl(ToSqlOptions options) {
if (isImplicit_) {
if (options.showImplictCasts()) {
- // for implicit casts, targetTypeDef_ is null
+ // for implicit casts, targetTypeDef_ and castFormat_ are null
return "CAST(" + getChild(0).toSql(options) + " AS " + type_.toSql() + ")";
} else {
return getChild(0).toSql(options);
}
}
+ String formatClause = "";
+ if (castFormat_ != null && !castFormat_.isEmpty()) {
+ formatClause = " FORMAT \"" + castFormat_ + "\"";
+ }
return "CAST(" + getChild(0).toSql(options) + " AS " + targetTypeDef_.toString()
- + ")";
+ + formatClause + ")";
}
@Override
@@ -205,6 +224,13 @@ public class CastExpr extends Expr {
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.FUNCTION_CALL;
+ // Sets cast_expr in case FORMAT clause was provided and this is a cast between a
+ // datetime and a string.
+ if (null != castFormat_ &&
+ (type_.isDateOrTimeType() && getChild(0).getType().isStringType() ||
+ type_.isStringType() && getChild(0).getType().isDateOrTimeType())) {
+ msg.cast_expr = new TCastExpr(castFormat_);
+ }
}
@Override
@@ -212,6 +238,7 @@ public class CastExpr extends Expr {
return Objects.toStringHelper(this)
.add("isImplicit", isImplicit_)
.add("target", type_)
+ .add("format", castFormat_)
.addValue(super.debugString())
.toString();
}
@@ -238,19 +265,32 @@ public class CastExpr extends Expr {
"Unsupported cast to complex type: " + type_.toSql());
}
- boolean readyForCharCast =
- children_.get(0).getType().getPrimitiveType() == PrimitiveType.STRING ||
- children_.get(0).getType().getPrimitiveType() == PrimitiveType.CHAR;
- if (type_.getPrimitiveType() == PrimitiveType.CHAR && !readyForCharCast) {
+ boolean twoStepCastNeeded =
+ type_.getPrimitiveType() == PrimitiveType.CHAR &&
+ children_.get(0).getType().getPrimitiveType() != PrimitiveType.STRING &&
+ children_.get(0).getType().getPrimitiveType() != PrimitiveType.CHAR;
+ if (twoStepCastNeeded) {
// Back end functions only exist to cast string types to CHAR, there is not a cast
// for every type since it is redundant with STRING. Casts to go through 2 casts:
// (1) cast to string, to stringify the value
// (2) cast to CHAR, to truncate or pad with spaces
- CastExpr tostring = new CastExpr(ScalarType.STRING, children_.get(0));
+ CastExpr tostring = new CastExpr(ScalarType.STRING, children_.get(0), castFormat_);
tostring.analyze();
children_.set(0, tostring);
}
+ if (null != castFormat_ && !twoStepCastNeeded) {
+ if (!(type_.isDateOrTimeType() && getChild(0).getType().isStringType()) &&
+ !(type_.isStringType() && getChild(0).getType().isDateOrTimeType())) {
+ // FORMAT clause works only for casting between date types and string types
+ throw new AnalysisException("FORMAT clause is not applicable from " +
+ getChild(0).getType() + " to " + type_);
+ }
+ if (castFormat_.isEmpty()) {
+ throw new AnalysisException("FORMAT clause can't be empty");
+ }
+ }
+
if (children_.get(0) instanceof NumericLiteral && type_.isFloatingPointType()) {
// Special case casting a decimal literal to a floating point number. The
// decimal literal can be interpreted as either and we want to avoid casts
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index 5fcb747..17d7f22 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -3236,4 +3236,67 @@ public class AnalyzeExprsTest extends AnalyzerTest {
Assert.assertNotNull(foundFn);
Assert.assertEquals(Type.TIMESTAMP, foundFn.getArgs()[0]);
}
+
+ @Test
+ public void TestCastFormatClauseFromString() throws AnalysisException {
+ AnalysisError("select cast('05-01-2017' AS DATETIME FORMAT 'MM-dd-yyyy')",
+ "Unsupported data type: DATETIME");
+ AnalysisError("select cast('05-01-2017' AS INT FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from STRING to INT");
+ AnalysisError("select cast('05-01-2017' AS STRING FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from STRING to STRING");
+ AnalysisError("select cast('05-01-2017' AS BOOLEAN FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from STRING to BOOLEAN");
+ AnalysisError("select cast('05-01-2017' AS DOUBLE FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from STRING to DOUBLE");
+ AnalysisError("select cast('05-01-2017' AS TIMESTAMP FORMAT '')",
+ "FORMAT clause can't be empty");
+ AnalyzesOk("select cast('05-01-2017' AS TIMESTAMP FORMAT 'MM-dd-yyyy')");
+ AnalyzesOk("select cast('05-01-2017' AS DATE FORMAT 'MM-dd-yyyy')");
+ }
+
+ @Test
+ public void TestCastFormatClauseFromDatetime() throws AnalysisException {
+ RunCastFormatTestOnType("TIMESTAMP");
+ RunCastFormatTestOnType("DATE");
+ }
+
+ private void RunCastFormatTestOnType(String type) {
+ String to_timestamp_cast = "cast('05-01-2017' as " + type + ")";
+ AnalysisError(
+ "select cast(" + to_timestamp_cast + " as DATETIME FORMAT 'MM-dd-yyyy')",
+ "Unsupported data type: DATETIME");
+ if (!type.equals("TIMESTAMP")) {
+ AnalysisError(
+ "select cast(" + to_timestamp_cast + " as TIMESTAMP FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from " + type + " to TIMESTAMP");
+ }
+ if (!type.equals("DATE")) {
+ AnalysisError("select cast(" + to_timestamp_cast + " as DATE FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from " + type + " to DATE");
+ }
+ AnalysisError("select cast(" + to_timestamp_cast + " AS INT FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from " + type +" to INT");
+ AnalysisError("select cast(" + to_timestamp_cast + " AS BOOLEAN FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from " + type + " to BOOLEAN");
+ AnalysisError("select cast(" + to_timestamp_cast + " AS DOUBLE FORMAT 'MM-dd-yyyy')",
+ "FORMAT clause is not applicable from " + type + " to DOUBLE");
+ AnalysisError("select cast(" + to_timestamp_cast + " AS STRING FORMAT '')",
+ "FORMAT clause can't be empty");
+ AnalyzesOk("select cast(" + to_timestamp_cast + " AS STRING FORMAT 'MM-dd-yyyy')");
+ AnalyzesOk("select cast(" + to_timestamp_cast + " AS VARCHAR FORMAT 'MM-dd-yyyy')");
+ AnalyzesOk("select cast(" + to_timestamp_cast + " AS CHAR(10) FORMAT 'MM-dd-yyyy')");
+ }
+
+ @Test
+ public void TestToStringOnCastFormatClause() throws AnalysisException {
+ String cast_str = "CAST('05-01-2017' AS TIMESTAMP FORMAT \"MM-dd-yyyy\")";
+ SelectStmt select = (SelectStmt) AnalyzesOk("select " + cast_str);
+ Assert.assertEquals(cast_str, select.getResultExprs().get(0).toSqlImpl());
+
+ cast_str = "CAST('05-01-2017' AS DATE FORMAT \"MM-dd-yyyy\")";
+ select = (SelectStmt) AnalyzesOk("select " + cast_str);
+ Assert.assertEquals(cast_str, select.getResultExprs().get(0).toSqlImpl());
+ }
+
}
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index f67df17..eebefc9 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -1455,8 +1455,14 @@ public class ParserTest extends FrontendTestBase {
public void TestCastExprs() {
ParsesOk("select cast(a + 5.0 as string) from t");
ParsesOk("select cast(NULL as string) from t");
+ ParsesOk("select cast('05-01-2017' as timestamp format 'MM-dd-yyyy')");
+ ParsesOk("select cast('2017.01.02' as date format 'YYYY-MM-DD')");
ParserError("select cast(a + 5.0 as badtype) from t");
ParserError("select cast(a + 5.0, string) from t");
+ ParserError("select cast('05-01-2017' as timestamp format 12345)");
+ ParserError("select cast('05-01-2017' as timestamp format )");
+ ParserError("select cast('05-01-2017' as timestamp format NULL)");
+ ParserError("select cast('05-01-2017' as date format NULL)");
}
@Test
diff --git a/testdata/workloads/functional-query/queries/QueryTest/cast_format_from_table.test b/testdata/workloads/functional-query/queries/QueryTest/cast_format_from_table.test
new file mode 100644
index 0000000..e4c878b
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/cast_format_from_table.test
@@ -0,0 +1,109 @@
+# Input value for the tests:
+# select timestamp_col from alltypes where id=30;
+# 2009-01-04 00:30:01.350000000
+====
+---- QUERY
+# timestamp to string conversion
+select
+ cast(timestamp_col as string format 'HH24 HH24 HH12:MI:SS MM-DD-YYYY DDD SSSSS P.M.')
+ from alltypes
+ where id=30
+---- RESULTS
+'00 00 12:30:01 01-04-2009 004 01801 A.M.'
+---- TYPES
+string
+====
+---- QUERY
+# timestamp to varchar conversion
+select
+ cast(timestamp_col as varchar format 'HH24 HH24 HH12:MI:SS MM-DD-YYYY DDD SSSSS PM')
+ from alltypes
+ where id=30
+---- RESULTS
+'00 00 12:30:01 01-04-2009 004 01801 AM'
+---- TYPES
+string
+====
+---- QUERY
+# timestamp to char conversion
+select
+ cast(timestamp_col as char(38) format 'HH24 HH24 HH12:MI:SS MM-DD-YYYY DDD SSSSS PM')
+ from alltypes
+ where id=30
+---- RESULTS
+'00 00 12:30:01 01-04-2009 004 01801 AM'
+---- TYPES
+char
+====
+---- QUERY
+# For the string types to timestamp conversions I use the timestamp_col from alltypes
+# convert it to the desired string format and then convert back to timestamp.
+# string to timestamp conversion
+select cast(cast(timestamp_col as string format 'DD-MM-YYYY HH12:MI:SS') as timestamp
+ format 'DD-MM-YYYY HH:MI:SS') from alltypes where id=30
+---- RESULTS
+2009-01-04 00:30:01
+---- TYPES
+timestamp
+====
+---- QUERY
+# varchar to timestamp conversion
+select cast(cast(timestamp_col as varchar format 'HH:MI:SS YYYY-MM-DD') as timestamp
+ format 'HH:MI:SS YYYY-MM-DD') from alltypes where id=30
+---- RESULTS
+2009-01-04 00:30:01
+---- TYPES
+timestamp
+====
+---- QUERY
+# char to timestamp conversion
+select cast(cast(timestamp_col as char(19) format 'yyyy/dd/mm hh:mi:ss') as timestamp
+ format 'yyyy/dd/mm hh:mi:ss') from alltypes where id=30
+---- RESULTS
+2009-01-04 00:30:01
+---- TYPES
+timestamp
+====
+---- QUERY
+# Multiple casts are in the expr tree with different format.
+# Input: '01/04/09' => 2009-04-01 > 2009-01-04
+select (cast(date_string_col as timestamp format 'DD/MM/YYYY') >
+ cast(date_string_col as timestamp format 'MM/DD/YYYY'))
+ from alltypes where id=30
+---- RESULTS
+True
+---- TYPES
+boolean
+====
+---- QUERY
+# Multiple nested casts are in the expr tree with different format.
+# Input: '01/04/09'
+select cast(cast(date_string_col as timestamp format 'mm/dd/yyyy')
+ as string format 'hh24:mi yyyy-mm-dd') from alltypes where id=30;
+---- RESULTS
+'00:00 2009-01-04'
+---- TYPES
+string
+====
+---- QUERY
+# Year boundary tests
+select cast("1399-12-31" as timestamp FORMAT "YYYY-MM-DD");
+---- RESULTS
+NULL
+---- TYPES
+timestamp
+====
+---- QUERY
+select cast("1400-01-01" as timestamp FORMAT "YYYY-MM-DD");
+---- RESULTS
+1400-01-01 00:00:00
+---- TYPES
+timestamp
+====
+---- QUERY
+select cast("9999-12-31" as timestamp FORMAT "YYYY-MM-DD");
+---- RESULTS
+9999-12-31 00:00:00
+---- TYPES
+timestamp
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/date.test b/testdata/workloads/functional-query/queries/QueryTest/date.test
index b0dcc41..7ab47aa 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/date.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/date.test
@@ -591,3 +591,291 @@ select cast(cast('23:59:59' as timestamp) as date);
---- CATCH
UDF ERROR: Timestamp to Date conversion failed. Timestamp has no date component.
====
+---- QUERY
+# Basic test for FORMAT clause in CAST().
+select cast("2014-11-01" as date format "YYYY-MM-DD");
+---- RESULTS
+2014-11-01
+---- TYPES
+DATE
+====
+---- QUERY
+# Shuffle the pattern elements in the input.
+select cast("02-11-2014" as date format "dd-mm-yyyy");
+---- RESULTS
+2014-11-02
+---- TYPES
+DATE
+====
+---- QUERY
+# Don't accept time tokens for date when format is provided.
+select cast("2014-11-03 10" as date format "YYYY-MM-DD HH12");
+---- CATCH
+PARSE ERROR: Time tokens provided with date type.
+====
+---- QUERY
+# Input doesn't match the format.
+select cast("2014-11-04" as date format "MM-DD-YYYY");
+---- CATCH
+UDF ERROR: String to Date parse failed.
+====
+---- QUERY
+# Interchangeability of separators.
+select cast("2014/11/05" as date format "YYYY-MM-DD");
+---- RESULTS
+2014-11-05
+---- TYPES
+DATE
+====
+---- QUERY
+select cast("2014/- ,11'/:-05" as date format "YYYY-MM-DD");
+---- RESULTS
+2014-11-05
+---- TYPES
+DATE
+====
+---- QUERY
+# Day of year tests.
+select cast("2014 001" as date format "YYYY DDD");
+---- RESULTS
+2014-01-01
+---- TYPES
+DATE
+====
+---- QUERY
+select cast("365 2014" as date format "DDD-YYYY");
+---- RESULTS
+2014-12-31
+---- TYPES
+DATE
+====
+---- QUERY
+select cast("366 2014" as date format "DDD-YYYY");
+---- CATCH
+UDF ERROR: String to Date parse failed.
+====
+---- QUERY
+# Leap year with day of year token.
+select cast("2000-366" as date format "YYYY DDD");
+---- RESULTS
+2000-12-31
+---- TYPES
+DATE
+====
+---- QUERY
+select cast("2000 367" as date format "YYYY DDD");
+---- CATCH
+UDF ERROR: String to Date parse failed.
+====
+----QUERY
+# Leap year with day of year token before 1400.
+select cast('0004 100' as date format "YYYY DDD");
+---- RESULTS
+0004-04-09
+---- TYPES
+DATE
+====
+---- QUERY
+# Test how missing year digits are filled from current year.
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("095-01-31" as date FORMAT "YYYY-MM-DD");
+---- RESULTS
+2095-01-31
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("95-01-30" as date FORMAT "YYYY-MM-DD");
+---- RESULTS
+2095-01-30
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("5-01-29" as date FORMAT "YYYY-MM-DD");
+---- RESULTS
+2015-01-29
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("95-01-28" as date FORMAT "YYY-MM-DD");
+---- RESULTS
+2095-01-28
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("5-01-27" as date FORMAT "YYY-MM-DD");
+---- RESULTS
+2015-01-27
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("5-01-26" as date FORMAT "YY-MM-DD");
+---- RESULTS
+2015-01-26
+---- TYPES
+DATE
+====
+---- QUERY
+# RRRR with 4-digit year falls back to YYYY
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("2017-01-25" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2017-01-25
+---- TYPES
+DATE
+====
+---- QUERY
+# RRRR with 3-digit year fills digits from current year
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("017-01-24" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2017-01-24
+---- TYPES
+DATE
+====
+---- QUERY
+# RRRR with 1-digit year fills digits from current year
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("7-01-23" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2017-01-23
+---- TYPES
+DATE
+====
+---- QUERY
+# RR with 1-digit year fills digits from current year
+SET NOW_STRING="2019-01-01 11:11:11";
+select cast("7-01-22" as date FORMAT "RR-MM-DD");
+---- RESULTS
+2017-01-22
+---- TYPES
+DATE
+====
+---- QUERY
+# Round year when last 2 digits of current year is less than 50
+SET NOW_STRING="2049-01-01 11:11:11";
+select cast("49-01-21" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2049-01-21
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2049-01-01 11:11:11";
+select cast("50-01-20" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+1950-01-20
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2000-01-01 11:11:11";
+select cast("49-01-19" as date FORMAT "RR-MM-DD");
+---- RESULTS
+2049-01-19
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2000-01-01 11:11:11";
+select cast("50-01-18" as date FORMAT "RR-MM-DD");
+---- RESULTS
+1950-01-18
+---- TYPES
+DATE
+====
+---- QUERY
+# Round year when last 2 digits of current year is greater than 49
+SET NOW_STRING="2050-01-01 11:11:11";
+select cast("49-01-17" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2149-01-17
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2050-01-01 11:11:11";
+select cast("50-01-16" as date FORMAT "RRRR-MM-DD");
+---- RESULTS
+2050-01-16
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2099-01-01 11:11:11";
+select cast("49-01-15" as date FORMAT "RR-MM-DD");
+---- RESULTS
+2149-01-15
+---- TYPES
+DATE
+====
+---- QUERY
+SET NOW_STRING="2099-01-01 11:11:11";
+select cast("50-01-14" as date FORMAT "RR-MM-DD");
+---- RESULTS
+2050-01-14
+---- TYPES
+DATE
+====
+---- QUERY
+# In a date to string cast round year act like regular 'YYYY' or 'YY' tokens.
+# select date_col from date_tbl where id_col = 4;
+# 2017-11-28
+select cast(date_col as string format "RRRR RR") from date_tbl where id_col = 4;
+---- RESULTS
+'2017 17'
+---- TYPES
+STRING
+====
+---- QUERY
+# Date to string conversion using all but round year tokens.
+select cast(date_col as string format "YYYY YYY YY Y MM DD DDD") from date_tbl
+where id_col = 4;
+---- RESULTS
+'2017 017 17 7 11 28 332'
+---- TYPES
+STRING
+====
+---- QUERY
+# Time tokens not allowed with date type.
+select cast(date_col as string format "YYYY-MM-DD HH12") from date_tbl
+where id_col = 4;
+---- CATCH
+PARSE ERROR: Time tokens provided with date type.
+====
+---- QUERY
+# Boundary tests with FORMAT clause
+select cast("0000-01-01" as date FORMAT "YYYY-MM-DD");
+---- RESULTS
+0000-01-01
+====
+---- QUERY
+select cast("9999-12-31" as date FORMAT "YYYY-MM-DD");
+---- RESULTS
+9999-12-31
+====
+---- QUERY
+# Cast NULL string to date
+select cast(cast(NULL as string) as date format "YYYY-MM-DD");
+---- RESULTS
+NULL
+---- TYPES
+DATE
+====
+---- QUERY
+# Cast NULL date to string
+select cast(cast(NULL as date) as string format "YYYY-MM-DD");
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
diff --git a/tests/query_test/test_cast_with_format.py b/tests/query_test/test_cast_with_format.py
new file mode 100644
index 0000000..8287f09
--- /dev/null
+++ b/tests/query_test/test_cast_with_format.py
@@ -0,0 +1,733 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.test_dimensions import create_beeswax_hs2_dimension
+
+
+class TestCastWithFormat(ImpalaTestSuite):
+ @classmethod
+ def get_workload(self):
+ return "functional-query"
+
+ # Run the basic tests once for Beeaswax and once for HS2. The underlying functionality
+ # is independent of the file format so make sense to pick one format for testing.
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestCastWithFormat, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format == 'parquet')
+
+ cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
+
+ def test_basic_inputs_from_table(self, vector):
+ self.run_test_case('QueryTest/cast_format_from_table', vector)
+
+ def test_basic_inputs_without_row(self, vector):
+ # Cast without format clause to cover the default format
+ result = self.client.execute("select cast('2017-05-01 01:23:45.678912345' as "
+ "timestamp)")
+ assert result.data == ["2017-05-01 01:23:45.678912345"]
+
+ # Basic input to cover a datetime with timezone scenario
+ result = self.client.execute("select cast('2017-05-03 08:59:01.123456789PM 01:30'"
+ "as timestamp FORMAT 'YYYY-MM-DD HH12:MI:SS.FF9PM TZH:TZM')")
+ assert result.data == ["2017-05-03 20:59:01.123456789"]
+
+ # Input that contains shuffled date without time
+ result = self.client.execute("select cast('12-2010-05' as timestamp format "
+ "'DD-YYYY-MM')")
+ assert result.data == ["2010-05-12 00:00:00"]
+
+ # Shuffle the input timestamp and the format clause
+ result = self.client.execute("select cast('59 04-30-2017-05 01PM 01:08.123456789'"
+ "as timestamp FORMAT 'MI DD-TZM-YYYY-MM TZHPM SS:HH12.FF9')")
+ assert result.data == ["2017-05-04 20:59:01.123456789"]
+
+ # Input and format without separators
+ # Note, 12:01 HH12 AM is 00:01 with the internal 0-23 representation.
+ result = self.client.execute("select cast('20170501120159123456789AM-0130' as "
+ "timestamp FORMAT 'YYYYDDMMHH12MISSFFAMTZHTZM')")
+ assert result.data == ["2017-01-05 00:01:59.123456789"]
+
+ # Shuffled input without separators
+ result = self.client.execute("select cast('59043020170501PM0108123456789'"
+ "as timestamp FORMAT 'MIDDTZMYYYYMMTZHPMSSHH12FF9')")
+ assert result.data == ["2017-05-04 20:59:01.123456789"]
+
+ # Separator section lengths differ between input and format
+ result = self.client.execute("select cast('--2017----05-01-' as "
+ "timestamp FORMAT '-YYYY--MM---DD---')")
+ assert result.data == ["2017-05-01 00:00:00"]
+
+ # If the input string has unprocessed tokens
+ result = self.client.execute("select cast('2017-05-01 12:30' as "
+ "timestamp FORMAT 'YYYY-MM-DD')")
+ assert result.data == ["NULL"]
+ result = self.client.execute("select cast('2017-05-01-12:30' as "
+ "timestamp FORMAT 'YYYY-MM-DD-')")
+ assert result.data == ["NULL"]
+
+ # If the format string has unprocessed tokens
+ result = self.client.execute("select cast('2017-05-01' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI')")
+ assert result.data == ["NULL"]
+ result = self.client.execute("select cast('2017-05-01-' as "
+ "timestamp FORMAT 'YYYY-MM-DD-HH12')")
+ assert result.data == ["NULL"]
+
+ # Timestamp to string types formatting
+ result = self.client.execute(
+ "select cast(cast('2012-11-04 13:02:59.123456' as timestamp) "
+ "as string format 'DD-MM-YYYY MI:HH12:SS A.M. FF9 DDD SSSSS HH12 HH24')")
+ assert result.data == ["04-11-2012 02:01:59 P.M. 123456000 309 46979 01 13"]
+
+ result = self.client.execute(
+ "select cast(cast('2012-11-04 13:02:59.123456' as timestamp) "
+ "as varchar format 'DD-MM-YYYY MI:HH12:SS A.M. FF9 DDD SSSSS HH12 HH24')")
+ assert result.data == ["04-11-2012 02:01:59 P.M. 123456000 309 46979 01 13"]
+
+ result = self.client.execute(
+ "select cast(cast('2012-11-04 13:02:59.123456' as timestamp) "
+ "as char(50) format 'DD-MM-YYYY MI:HH12:SS A.M. FF9 DDD SSSSS HH12 HH24')")
+ assert result.data == ["04-11-2012 02:01:59 P.M. 123456000 309 46979 01 13"]
+
+ # Cast NULL string to timestamp
+ result = self.client.execute("select cast(cast(NULL as string) as timestamp "
+ "FORMAT 'YYYY-MM-DD')")
+ assert result.data == ["NULL"]
+
+ # Cast NULL timestamp to string
+ result = self.client.execute("select cast(cast(NULL as timestamp) as string "
+ "FORMAT 'YYYY-MM-DD')")
+ assert result.data == ["NULL"]
+
+ def test_iso8601_format(self):
+ # Basic string to timestamp scenario
+ result = self.client.execute("select cast('2018-11-10T15:11:04Z' as "
+ "timestamp FORMAT 'YYYY-MM-DDTHH24:MI:SSZ')")
+ assert result.data == ["2018-11-10 15:11:04"]
+
+ # ISO8601 format elements are case-insensitive
+ result = self.client.execute("select cast('2018-11-09t15:11:04Z' as "
+ "timestamp FORMAT 'YYYY-MM-DDTHH24:MI:SSz')")
+ assert result.data == ["2018-11-09 15:11:04"]
+
+ result = self.client.execute("select cast('2018-11-08T15:11:04z' as "
+ "timestamp FORMAT 'YYYY-MM-DDtHH24:MI:SSZ')")
+ assert result.data == ["2018-11-08 15:11:04"]
+
+ # Format path
+ result = self.client.execute("select cast(cast('2018-11-10 15:11:04' as "
+ "timestamp) as string format 'YYYY-MM-DDTHH24:MI:SSZ')")
+ assert result.data == ["2018-11-10T15:11:04Z"]
+
+ def test_lowercase_format_elements(self):
+ result = self.client.execute("select cast('2019-11-20 15:59:44.123456789 01:01' as "
+ "timestamp format 'yyyy-mm-dd hh24:mi:ss.ff9 tzh-tzm')")
+ assert result.data == ["2019-11-20 15:59:44.123456789"]
+
+ result = self.client.execute("select cast('2019-300 15:59:44.123456789 01:01' as "
+ "timestamp format 'yyyy-ddd hh24:mi:ss.ff9 tzh-tzm')")
+ assert result.data == ["2019-10-27 15:59:44.123456789"]
+
+ result = self.client.execute("select cast('2019-11-21 11:59:44.123456789 p.m. 01:01' "
+ "as timestamp format 'yyyy-mm-dd hh12:mi:ss.ff9 am tzh-tzm')")
+ assert result.data == ["2019-11-21 23:59:44.123456789"]
+
+ result = self.client.execute("select cast('2019-11-22 10000.123456789 02:02' "
+ "as timestamp format 'yyyy-mm-dd sssss ff9 tzh-tzm')")
+ assert result.data == ["2019-11-22 02:46:40.123456789"]
+
+ def test_year(self):
+ # Test lower boundary of year
+ result = self.client.execute("select cast('1399-05-01' as "
+ "timestamp FORMAT 'YYYY-MM-DD')")
+ assert result.data == ["NULL"]
+
+ # YYYY with less than 4 digits in the input
+ query_options = dict({'now_string': '2019-01-01 11:11:11'})
+
+ result = self.execute_query("select cast('095-01-31' as "
+ "timestamp FORMAT 'YYYY-MM-DD')", query_options)
+ assert result.data == ["2095-01-31 00:00:00"]
+
+ result = self.execute_query("select cast('95-02-28' as "
+ "timestamp FORMAT 'YYYY-MM-DD')", query_options)
+ assert result.data == ["2095-02-28 00:00:00"]
+
+ result = self.execute_query("select cast('5-03-31' as "
+ "timestamp FORMAT 'YYYY-MM-DD')", query_options)
+ assert result.data == ["2015-03-31 00:00:00"]
+
+ # YYY with less than 3 digits in the input
+ result = self.execute_query("select cast('95-04-30' as "
+ "timestamp FORMAT 'YYY-MM-DD')", query_options)
+ assert result.data == ["2095-04-30 00:00:00"]
+
+ result = self.execute_query("select cast('5-05-31' as "
+ "timestamp FORMAT 'YYY-MM-DD')", query_options)
+ assert result.data == ["2015-05-31 00:00:00"]
+
+ # YY with 1 digits in the input
+ result = self.execute_query("select cast('5-06-30' as "
+ "timestamp FORMAT 'YY-MM-DD')", query_options)
+ assert result.data == ["2015-06-30 00:00:00"]
+
+ # YYY, YY, Y tokens without separators
+ result = self.execute_query("select cast('0950731' as "
+ "timestamp FORMAT 'YYYMMDD')", query_options)
+ assert result.data == ["2095-07-31 00:00:00"]
+
+ result = self.execute_query("select cast('950831' as "
+ "timestamp FORMAT 'YYMMDD')", query_options)
+ assert result.data == ["2095-08-31 00:00:00"]
+
+ result = self.execute_query("select cast('50930' as "
+ "timestamp FORMAT 'YMMDD')", query_options)
+ assert result.data == ["2015-09-30 00:00:00"]
+
+ # Timestamp to string formatting
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'YYYY')", query_options)
+ assert result.data == ["2019"]
+
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'YYY')", query_options)
+ assert result.data == ["019"]
+
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'YY')", query_options)
+ assert result.data == ["19"]
+
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'Y')", query_options)
+ assert result.data == ["9"]
+
+ def test_round_year(self):
+ query_options = dict({'now_string': '2019-01-01 11:11:11'})
+
+ # Test lower boundar of round year
+ result = self.client.execute("select cast('1399-05-01' as "
+ "timestamp FORMAT 'RRRR-MM-DD')")
+ assert result.data == ["NULL"]
+
+ result = self.client.execute("select cast('1400-05-21' as "
+ "timestamp FORMAT 'RRRR-MM-DD')")
+ assert result.data == ["1400-05-21 00:00:00"]
+
+ # RRRR with 4-digit year falls back to YYYY
+ result = self.execute_query("select cast('2017-05-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2017-05-31 00:00:00"]
+
+ # RRRR with 3-digit year fills digits from current year
+ result = self.execute_query("select cast('017-01-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2017-01-31 00:00:00"]
+
+ # RRRR wit 1-digit year fills digits from current year
+ result = self.execute_query("select cast('0-07-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2010-07-31 00:00:00"]
+
+ # RR with 1-digit year fills digits from current year
+ result = self.execute_query("select cast('9-08-31' as "
+ "timestamp FORMAT 'RR-MM-DD')", query_options)
+ assert result.data == ["2019-08-31 00:00:00"]
+
+ # Round year when last 2 digits of current year is less than 50
+ query_options = dict({'now_string': '2049-01-01 11:11:11'})
+ result = self.execute_query("select cast('49-03-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2049-03-31 00:00:00"]
+
+ result = self.execute_query("select cast('50-03-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["1950-03-31 00:00:00"]
+
+ query_options = dict({'now_string': '2000-01-01 11:11:11'})
+ result = self.execute_query("select cast('49-03-31' as "
+ "timestamp FORMAT 'RR-MM-DD')", query_options)
+ assert result.data == ["2049-03-31 00:00:00"]
+
+ result = self.execute_query("select cast('50-03-31' as "
+ "timestamp FORMAT 'RR-MM-DD')", query_options)
+ assert result.data == ["1950-03-31 00:00:00"]
+
+ # Round year when last 2 digits of current year is greater than 49
+ query_options = dict({'now_string': '2050-01-01 11:11:11'})
+ result = self.execute_query("select cast('49-03-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2149-03-31 00:00:00"]
+
+ result = self.execute_query("select cast('50-03-31' as "
+ "timestamp FORMAT 'RRRR-MM-DD')", query_options)
+ assert result.data == ["2050-03-31 00:00:00"]
+
+ query_options = dict({'now_string': '2099-01-01 11:11:11'})
+ result = self.execute_query("select cast('49-03-31' as "
+ "timestamp FORMAT 'RR-MM-DD')", query_options)
+ assert result.data == ["2149-03-31 00:00:00"]
+
+ result = self.execute_query("select cast('50-03-31' as "
+ "timestamp FORMAT 'RR-MM-DD')", query_options)
+ assert result.data == ["2050-03-31 00:00:00"]
+
+ # In a datetime to sting cast round year act like regular 'YYYY' or 'YY' tokens.
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'RRRR')", query_options)
+ assert result.data == ["2019"]
+
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'RR')", query_options)
+ assert result.data == ["19"]
+
+ def test_day_in_year(self):
+ # Test "day in year" token in a non leap year scenario
+ result = self.execute_query("select cast('2019 1' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2019-01-01 00:00:00"]
+
+ result = self.execute_query("select cast('2019 31' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2019-01-31 00:00:00"]
+
+ result = self.execute_query("select cast('2019 32' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2019-02-01 00:00:00"]
+
+ result = self.execute_query("select cast('2019 60' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2019-03-01 00:00:00"]
+
+ result = self.execute_query("select cast('2019 365' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2019-12-31 00:00:00"]
+
+ result = self.execute_query("select cast('2019 366' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["NULL"]
+
+ # Test "day in year" token in a leap year scenario
+ result = self.execute_query("select cast('2000 60' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2000-02-29 00:00:00"]
+
+ result = self.execute_query("select cast('2000 61' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2000-03-01 00:00:00"]
+
+ result = self.execute_query("select cast('2000 366' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["2000-12-31 00:00:00"]
+
+ result = self.execute_query("select cast('2000 367' as timestamp FORMAT 'YYYY DDD')")
+ assert result.data == ["NULL"]
+
+ # Test "day in year" token without separators
+ result = self.execute_query("select cast('20190011120' as timestamp "
+ "FORMAT 'YYYYDDDHH12MI')")
+ assert result.data == ["2019-01-01 11:20:00"]
+
+ # Timestamp to string formatting
+ result = self.execute_query("select cast(cast('2019-01-01' as timestamp) as string "
+ "format'DDD')")
+ assert result.data == ["001"]
+
+ result = self.execute_query("select cast(cast('2019-12-31' as timestamp) as string "
+ "format'DDD')")
+ assert result.data == ["365"]
+
+ result = self.execute_query("select cast(cast('2000-12-31' as timestamp) as string "
+ "format'DDD')")
+ assert result.data == ["366"]
+
+ result = self.execute_query("select cast(cast('2019 123' as timestamp "
+ "format 'YYYY DDD') as string format'DDD')")
+ assert result.data == ["123"]
+
+ def test_second_of_day(self):
+ # Check boundaries
+ result = self.client.execute("select cast('2019-11-10 86399.11' as "
+ "timestamp FORMAT 'YYYY-MM-DD SSSSS.FF2')")
+ assert result.data == ["2019-11-10 23:59:59.110000000"]
+
+ result = self.client.execute("select cast('2019-11-10 0' as "
+ "timestamp FORMAT 'YYYY-MM-DD SSSSS')")
+ assert result.data == ["2019-11-10 00:00:00"]
+
+ # Without separators full 5-digit "second of day" has to be given
+ result = self.client.execute("select cast('11-10 036612019' as "
+ "timestamp FORMAT 'MM-DD SSSSSYYYY')")
+ assert result.data == ["2019-11-10 01:01:01"]
+
+ # Check timezone offsets with "second of day"
+ result = self.client.execute("select cast('2019-11-10 036611010' as "
+ "timestamp FORMAT 'YYYY-MM-DD SSSSSTZHTZM')")
+ assert result.data == ["2019-11-10 01:01:01"]
+
+ # Timestamp to string formatting
+ result = self.client.execute("select cast(cast('2019-01-01 01:01:01' as timestamp) "
+ "as string format 'SSSSS')")
+ assert result.data == ["03661"]
+
+ result = self.client.execute("select cast(cast('2019-01-01' as timestamp) as string "
+ "format 'SSSSS')")
+ assert result.data == ["00000"]
+
+ result = self.client.execute("select cast(cast('2019-01-01 23:59:59' as timestamp) "
+ "as string format 'SSSSS')")
+ assert result.data == ["86399"]
+
+ def test_fraction_seconds(self):
+ result = self.execute_query("select cast('2019-11-08 123456789' as "
+ "timestamp FORMAT 'YYYY-MM-DD FF9')")
+ assert result.data == ["2019-11-08 00:00:00.123456789"]
+
+ result = self.execute_query("select cast('2019-11-08 1' as "
+ "timestamp FORMAT 'YYYY-MM-DD FF')")
+ assert result.data == ["2019-11-08 00:00:00.100000000"]
+
+ result = self.execute_query("select cast('2019-11-08 1234567890' as "
+ "timestamp FORMAT 'YYYY-MM-DD FF')")
+ assert result.data == ["NULL"]
+
+ result = self.execute_query("select cast('2019-11-08' as "
+ "timestamp FORMAT 'YYYY-MM-DD FF')")
+ assert result.data == ["NULL"]
+
+ self.run_fraction_test(1)
+ self.run_fraction_test(2)
+ self.run_fraction_test(3)
+ self.run_fraction_test(4)
+ self.run_fraction_test(5)
+ self.run_fraction_test(6)
+ self.run_fraction_test(7)
+ self.run_fraction_test(8)
+ self.run_fraction_test(9)
+
+ def run_fraction_test(self, length):
+ MAX_LENGTH = 9
+ fraction_part = ""
+ for x in range(length):
+ fraction_part += str(x + 1)
+ template_input = "select cast('2019-11-08 %s' as timestamp FORMAT 'YYYY-MM-DD FF%s')"
+ input_str = template_input % (fraction_part, length)
+
+ expected = "2019-11-08 00:00:00." + fraction_part + ("0" * (MAX_LENGTH - length))
+ result = self.execute_query(input_str)
+ assert result.data == [expected]
+
+ input2_str = template_input % (fraction_part + str(length + 1), length)
+ result = self.execute_query(input2_str)
+ assert result.data == ["NULL"]
+
+ def test_meridiem_indicator(self):
+ # Check 12 hour diff between AM and PM
+ result = self.client.execute("select cast('2017-05-03 08 AM' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12 AM')")
+ assert result.data == ["2017-05-03 08:00:00"]
+
+ result = self.client.execute("select cast('2017-05-04 08 PM' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12 PM')")
+ assert result.data == ["2017-05-04 20:00:00"]
+
+ # Check that any meridiem indicator in the pattern matches any meridiem indicator in
+ # the input
+ result = self.client.execute("select cast('2017-05-05 12AM' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12PM')")
+ assert result.data == ["2017-05-05 00:00:00"]
+
+ result = self.client.execute("select cast('2017-05-06 P.M.12' as "
+ "timestamp FORMAT 'YYYY-MM-DD AMHH12')")
+ assert result.data == ["2017-05-06 12:00:00"]
+
+ result = self.client.execute("select cast('2017-05-07 PM 01' as "
+ "timestamp FORMAT 'YYYY-MM-DD A.M. HH12')")
+ assert result.data == ["2017-05-07 13:00:00"]
+
+ # Test lowercase indicator in input
+ result = self.client.execute("select cast('2017-05-08 pm09' as "
+ "timestamp FORMAT 'YYYY-MM-DD P.M.HH12')")
+ assert result.data == ["2017-05-08 21:00:00"]
+
+ result = self.client.execute("select cast('2017-05-09 10a.m.' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12PM')")
+ assert result.data == ["2017-05-09 10:00:00"]
+
+ # Test that '.' in indicator doesn't conflict with '.' as separator
+ result = self.client.execute("select cast('2017-05-11 9.AM.10' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12.P.M..MI')")
+ assert result.data == ["2017-05-11 09:10:00"]
+
+ result = self.client.execute("select cast('2017-05-10.P.M..10' as "
+ "timestamp FORMAT 'YYYY-MM-DD.AM.HH12')")
+ assert result.data == ["2017-05-10 22:00:00"]
+
+ # Timestamp to string formatting
+ result = self.client.execute("select cast(cast('2019-01-01 00:15:10' as timestamp) "
+ "as string format 'HH12 P.M.')")
+ assert result.data == ["12 A.M."]
+
+ result = self.client.execute("select cast(cast('2019-01-01 12:15:10' as timestamp) "
+ "as string format 'HH12 AM')")
+ assert result.data == ["12 PM"]
+
+ result = self.client.execute("select cast(cast('2019-01-01 13:15:10' as timestamp) "
+ "as string format 'HH12 a.m.')")
+ assert result.data == ["01 p.m."]
+
+ result = self.client.execute("select cast(cast('2019-01-01 23:15:10' as timestamp) "
+ "as string format 'HH12 p.m.')")
+ assert result.data == ["11 p.m."]
+
+ def test_timezone_offsets(self):
+ # Test positive timezone offset.
+ result = self.client.execute("select cast('2018-01-01 10:00 AM +15:59' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-01-01 10:00:00"]
+
+ # Test negative timezone offset.
+ result = self.client.execute("select cast('2018-12-31 08:00 PM -15:59' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-12-31 20:00:00"]
+
+ # Minus sign before TZM.
+ result = self.client.execute("select cast('2018-12-31 08:00 AM 01:-59' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ # Minus sign right before one digit TZH.
+ result = self.client.execute("select cast('2018-12-31 08:00 AM--1:10' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ result = self.client.execute("select cast('2018-12-31 08:00 AM-5:00' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M.TZH:TZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ # One digit negative TZH at the end of the input string.
+ result = self.client.execute("select cast('2018-12-31 12:01 -1' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI TZH')")
+ assert result.data == ["2018-12-31 12:01:00"]
+
+ # Test timezone offset parsing without separators
+ result = self.client.execute("select cast('201812310800AM+0515' as "
+ "timestamp FORMAT 'YYYYMMDDHH12MIA.M.TZHTZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ result = self.client.execute("select cast('201812310800AM0515' as "
+ "timestamp FORMAT 'YYYYMMDDHH12MIA.M.TZHTZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ result = self.client.execute("select cast('201812310800AM-0515' as "
+ "timestamp FORMAT 'YYYYMMDDHH12MIA.M.TZHTZM')")
+ assert result.data == ["2018-12-31 08:00:00"]
+
+ # Test signed zero TZH with not null TZM
+ result = self.client.execute("select cast('2018-01-01 10:00 AM +00:59' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-01-01 10:00:00"]
+
+ result = self.client.execute("select cast('2018-01-01 10:00 AM -00:59' as "
+ "timestamp FORMAT 'YYYY-MM-DD HH12:MI A.M. TZH:TZM')")
+ assert result.data == ["2018-01-01 10:00:00"]
+
+ # Shuffle TZH and TZM into other elements
+ result = self.client.execute("select cast('2018-01-01 15 10:00 1 AM' as "
+ "timestamp FORMAT 'YYYY-MM-DD TZM HH12:MI TZH A.M.')")
+ assert result.data == ["2018-01-01 10:00:00"]
+
+ result = self.client.execute("select cast('2018-01-011510:00-01AM' as "
+ "timestamp FORMAT 'YYYY-MM-DDTZMHH12:MITZHA.M.')")
+ assert result.data == ["2018-01-01 10:00:00"]
+
+ # Timezone offset with default time
+ result = self.client.execute("select cast('2018-01-01 01:30' as timestamp "
+ "FORMAT 'YYYY-MM-DD TZH:TZM')")
+ assert result.data == ["2018-01-01 00:00:00"]
+
+ # Single minus sign before two digit TZH.
+ result = self.client.execute("select cast('2018-09-11 15:30:10-10' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI:SS-TZH')")
+ assert result.data == ["2018-09-11 15:30:10"]
+
+ # Non-digit TZH and TZM.
+ result = self.client.execute("select cast('2018-09-11 17:30:10 ab:10' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI:SS TZH:TZM')")
+ assert result.data == ["NULL"]
+
+ result = self.client.execute("select cast('2018-09-11 17:30:10 -ab:10' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI:SS TZH:TZM')")
+ assert result.data == ["NULL"]
+
+ result = self.client.execute("select cast('2018-09-11 17:30:10 +ab:10' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI:SS TZH:TZM')")
+ assert result.data == ["NULL"]
+
+ result = self.client.execute("select cast('2018-09-11 18:30:10 10:ab' as timestamp "
+ "FORMAT 'YYYY-MM-DD HH24:MI:SS TZH:TZM')")
+ assert result.data == ["NULL"]
+
+ def test_format_parse_errors(self):
+ # Invalid format
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'XXXX-dd-MM')")
+ assert "Bad date/time conversion format: XXXX-dd-MM" in str(err)
+
+ # Invalid use of SimpleDateFormat
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01 15:10' as timestamp format 'yyyy-MM-dd +hh:mm')")
+ assert "Bad date/time conversion format: yyyy-MM-dd +hh:mm" in str(err)
+
+ # Duplicate format element
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD MM')")
+ assert "Invalid duplication of format element" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD-YYYY')")
+ assert "Invalid duplication of format element" in str(err)
+
+ # Multiple year token provided
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD-YY')")
+ assert "Multiple year token provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYY-MM-DD-Y')")
+ assert "Multiple year token provided" in str(err)
+
+ # Year and round year conflict
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YY-MM-DD-RRRR')")
+ assert "Both year and round year are provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'RR-MM-DD-YYY')")
+ assert "Both year and round year are provided" in str(err)
+
+ # Day of year conflict
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DDD')")
+ assert "Day of year provided with day or month token" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-DD-DDD')")
+ assert "Day of year provided with day or month token" in str(err)
+
+ # Conflict between hour tokens
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH:HH24')")
+ assert "Multiple hour tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH12:HH24')")
+ assert "Multiple hour tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH12:HH')")
+ assert "Multiple hour tokens provided" in str(err)
+
+ # Conflict with median indicator
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD AM HH:MI A.M.')")
+ assert "Multiple median indicator tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD PM HH:MI am')")
+ assert "Multiple median indicator tokens provided" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH24:MI a.m.')")
+ assert "Conflict between median indicator and hour token" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD p.m.')")
+ assert "Missing hour token" in str(err)
+
+ # Conflict with second of day
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD SSSSS HH')")
+ assert "Second of day token conflicts with other token(s)" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH12:SSSSS')")
+ assert "Second of day token conflicts with other token(s)" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD HH24SSSSS')")
+ assert "Second of day token conflicts with other token(s)" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD MI SSSSS')")
+ assert "Second of day token conflicts with other token(s)" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD SS SSSSS')")
+ assert "Second of day token conflicts with other token(s)" in str(err)
+
+ # Too long format
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format '" +
+ "{char: <101}".format(char="s") + "')")
+ assert "The input format is too long" in str(err)
+
+ # Timezone offsets in a datetime to string formatting
+ err = self.execute_query_expect_failure(self.client,
+ "select cast(cast('2017-05-01 01:15' as timestamp format 'YYYY-MM-DD TZH:TZM') "
+ "as string format 'TZH')")
+ assert "Timezone offset not allowed in a datetime to string conversion" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast(cast('2017-05-01 01:15' as timestamp format 'YYYY-MM-DD TZH:TZM') "
+ "as string format 'TZM')")
+ assert "Timezone offset not allowed in a datetime to string conversion" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast(cast('2017-05-01 01:15' as timestamp format 'YYYY-MM-DD TZH:TZM') "
+ "as string format 'YYYY-MM-DD HH24:MI:SS TZH:TZM')")
+ assert "Timezone offset not allowed in a datetime to string conversion" in str(err)
+
+ # TZM requires TZH
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-12-31 08:00 AM 59' as timestamp FORMAT "
+ "'YYYY-MM-DD HH12:MI A.M. TZM')")
+ assert "TZH token is required for TZM" in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-12-31 08:00 AM -59' as timestamp FORMAT "
+ "'YYYY-MM-DD HH12:MI A.M. TZM')")
+ assert "TZH token is required for TZM" in str(err)
+
+ # Multiple fraction second token conflict
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-10' as timestamp format 'FF FF1')")
+ assert "Multiple fractional second token provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-10' as timestamp format 'FF2 FF3')")
+ assert "Multiple fractional second token provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-10' as timestamp format 'FF4 FF5')")
+ assert "Multiple fractional second token provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-10' as timestamp format 'FF6 FF7')")
+ assert "Multiple fractional second token provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2018-10-10' as timestamp format 'FF8 FF9')")
+ assert "Multiple fractional second token provided." in str(err)
+
+ # Verify that conflict check is not skipped when format ends with separators.
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2017-05-01' as timestamp format 'YYYY-MM-DD-RR--')")
+ assert "Both year and round year are provided" in str(err)