You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/07/08 20:56:19 UTC
[impala] 01/02: IMPALA-9531: Dropped support for dateless timestamps
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1bafb7bd29f4ecf1706d35e274c2b701a32281ac
Author: Adam Tamas <ta...@cloudera.com>
AuthorDate: Tue Apr 7 10:07:47 2020 +0200
IMPALA-9531: Dropped support for dateless timestamps
Removed the support for dateless timestamps.
During dateless timestamp casts if the format doesn't contain
date part we get an error during tokenization of the format.
If the input str doesn't contain a date part then we get null result.
Examples:
select cast('01:02:59' as timestamp);
This will come back as NULL value.
select to_timestamp('01:01:01', 'HH:mm:ss');
select cast('01:02:59' as timestamp format 'HH12:MI:SS');
select cast('12 AM' as timestamp FORMAT 'AM.HH12');
These will come back with a parsing errors.
Casting from a table will generate similar results.
Testing:
Modified the previous tests related to dateless timestamps.
Added test to read fromtables which are still containing dateless
timestamps and covered timestamp to string path when no date tokens
are requested in the output string.
Change-Id: I48c49bf027cc4b917849b3d58518facba372b322
Reviewed-on: http://gerrit.cloudera.org:8080/15866
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Gabor Kaszab <ga...@cloudera.com>
---
be/src/benchmarks/convert-timestamp-benchmark.cc | 2 +-
be/src/benchmarks/parse-timestamp-benchmark.cc | 4 +-
be/src/exec/text-converter.inline.h | 2 +-
be/src/exprs/cast-functions-ir.cc | 6 -
be/src/exprs/expr-test.cc | 35 +-----
be/src/exprs/scalar-expr-evaluator.cc | 2 +-
be/src/exprs/timestamp-functions-ir.cc | 12 +-
be/src/exprs/timestamp-functions.cc | 18 ++-
be/src/exprs/timestamp-functions.h | 6 +-
be/src/runtime/date-parse-util.cc | 2 +-
be/src/runtime/date-test.cc | 6 +-
.../runtime/datetime-iso-sql-format-tokenizer.cc | 3 +
be/src/runtime/datetime-parser-common.cc | 3 +
be/src/runtime/datetime-parser-common.h | 1 +
.../runtime/datetime-simple-date-format-parser.cc | 136 ++++++++-------------
.../runtime/datetime-simple-date-format-parser.h | 10 +-
be/src/runtime/timestamp-parse-util.cc | 6 +-
be/src/runtime/timestamp-test.cc | 67 +++++-----
be/src/runtime/timestamp-value.h | 3 +-
bin/rat_exclude_files.txt | 1 +
common/function-registry/impala_functions.py | 10 +-
.../apache/impala/analysis/AnalyzeKuduDDLTest.java | 3 +-
testdata/data/README | 12 ++
testdata/data/dateless_timestamps.parq | Bin 0 -> 435 bytes
testdata/data/dateless_timestamps.txt | 7 ++
testdata/data/lazy_timestamp.csv | 7 --
.../functional-query/queries/QueryTest/date.test | 7 --
.../QueryTest/dateless_timestamp_parquet.test | 25 ++++
.../queries/QueryTest/dateless_timestamp_text.test | 29 +++++
.../functional-query/queries/QueryTest/exprs.test | 20 ++-
.../queries/QueryTest/select-lazy-timestamp.test | 7 --
tests/data_errors/test_data_errors.py | 4 +-
tests/query_test/test_cast_with_format.py | 23 +++-
tests/query_test/test_scanners.py | 24 ++++
34 files changed, 275 insertions(+), 228 deletions(-)
diff --git a/be/src/benchmarks/convert-timestamp-benchmark.cc b/be/src/benchmarks/convert-timestamp-benchmark.cc
index a1d9331..c263fcf 100644
--- a/be/src/benchmarks/convert-timestamp-benchmark.cc
+++ b/be/src/benchmarks/convert-timestamp-benchmark.cc
@@ -166,7 +166,7 @@ fast path speedup: 10.2951
vector<TimestampValue> AddTestDataDateTimes(int n, const string& startstr) {
DateTimeFormatContext dt_ctx;
dt_ctx.Reset("yyyy-MMM-dd HH:mm:ss");
- SimpleDateFormatTokenizer::Tokenize(&dt_ctx);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE);
random_device rd;
mt19937 gen(rd());
diff --git a/be/src/benchmarks/parse-timestamp-benchmark.cc b/be/src/benchmarks/parse-timestamp-benchmark.cc
index c7ca51a..8d42311 100644
--- a/be/src/benchmarks/parse-timestamp-benchmark.cc
+++ b/be/src/benchmarks/parse-timestamp-benchmark.cc
@@ -258,9 +258,9 @@ int main(int argc, char **argv) {
timestamp_suite.AddBenchmark("Impala", TestImpalaSimpleDateFormat, ×);
dt_ctx_simple_date_format.Reset("yyyy-MM-dd HH:mm:ss", 19);
- SimpleDateFormatTokenizer::Tokenize(&dt_ctx_simple_date_format);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx_simple_date_format, PARSE);
dt_ctx_tz_simple_date_format.Reset("yyyy-MM-dd HH:mm:ss+hh:mm", 25);
- SimpleDateFormatTokenizer::Tokenize(&dt_ctx_tz_simple_date_format);
+ SimpleDateFormatTokenizer::Tokenize(&dt_ctx_tz_simple_date_format, PARSE);
dt_ctx_iso_sql_format.Reset("YYYY-MM-DD HH24:MI:SS", 21);
datetime_parse_util::IsoSqlFormatTokenizer tokenizer(&dt_ctx_iso_sql_format,
diff --git a/be/src/exec/text-converter.inline.h b/be/src/exec/text-converter.inline.h
index 07e8b9e..e3bee97 100644
--- a/be/src/exec/text-converter.inline.h
+++ b/be/src/exec/text-converter.inline.h
@@ -136,7 +136,7 @@ inline bool TextConverter::WriteSlot(const SlotDescriptor* slot_desc, Tuple* tup
case TYPE_TIMESTAMP: {
TimestampValue* ts_slot = reinterpret_cast<TimestampValue*>(slot);
*ts_slot = TimestampValue::ParseSimpleDateFormat(data, len);
- if (!ts_slot->HasDateOrTime()) {
+ if (!ts_slot->HasDate()) {
parse_result = StringParser::PARSE_FAILURE;
}
break;
diff --git a/be/src/exprs/cast-functions-ir.cc b/be/src/exprs/cast-functions-ir.cc
index 3a59cb4..84a899b 100644
--- a/be/src/exprs/cast-functions-ir.cc
+++ b/be/src/exprs/cast-functions-ir.cc
@@ -369,12 +369,6 @@ DateVal CastFunctions::CastToDateVal(FunctionContext* ctx, const StringVal& val)
DateVal CastFunctions::CastToDateVal(FunctionContext* ctx, const TimestampVal& val) {
if (val.is_null) return DateVal::null();
TimestampValue tv = TimestampValue::FromTimestampVal(val);
- if (UNLIKELY(!tv.HasDate())) {
- ctx->SetError("Timestamp to Date conversion failed. "
- "Timestamp has no date component.");
- return DateVal::null();
- }
-
DateValue dv(tv.DaysSinceUnixEpoch());
return dv.ToDateVal();
}
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index bc78015..4217ec8 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -1299,7 +1299,7 @@ void ExprTest::TestTimestampValue(const string& expr, const TimestampValue& expe
// We use this function for tests where the expected value is unknown, e.g., now().
void ExprTest::TestValidTimestampValue(const string& expr) {
EXPECT_TRUE(
- ConvertValue<TimestampValue>(GetValue(expr, TYPE_TIMESTAMP)).HasDateOrTime());
+ ConvertValue<TimestampValue>(GetValue(expr, TYPE_TIMESTAMP)).HasDate());
}
// We can't put this into TestValue() because GTest can't resolve
@@ -6772,6 +6772,8 @@ TEST_P(ExprTest, TimestampFunctions) {
+ 60*60*24), 'yyyy-MM-dd')", "1999-01-02");
TestStringValue("from_timestamp(to_timestamp(unix_timestamp('1999-01-01 10:10:10') \
+ 10), 'yyyy-MM-dd HH:mm:ss')", "1999-01-01 10:10:20");
+ TestStringValue("from_timestamp(cast('2999-05-05 11:11:11' as timestamp), 'HH:mm:ss')",
+ "11:11:11");
TestValue("cast('2011-12-22 09:10:11.123456789' as timestamp) > \
cast('2011-12-22 09:10:11.12345678' as timestamp)", TYPE_BOOLEAN, true);
TestValue("cast('2011-12-22 08:10:11.123456789' as timestamp) > \
@@ -6807,10 +6809,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestValue("dayofweek(cast('2011-12-18' as timestamp))", TYPE_INT, 1);
TestValue("dayofweek(cast('2011-12-22' as timestamp))", TYPE_INT, 5);
TestValue("dayofweek(cast('2011-12-24' as timestamp))", TYPE_INT, 7);
- TestValue("hour(cast('09:10:11.000000' as timestamp))", TYPE_INT, 9);
- TestValue("minute(cast('09:10:11.000000' as timestamp))", TYPE_INT, 10);
- TestValue("second(cast('09:10:11.000000' as timestamp))", TYPE_INT, 11);
- TestValue("millisecond(cast('09:10:11.123456' as timestamp))", TYPE_INT, 123);
TestStringValue(
"to_date(cast('2011-12-22 09:10:11.12345678' as timestamp))", "2011-12-22");
@@ -6821,7 +6819,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestValue("timestamp_cmp('1966-09-04 15:33:45','1966-05-04 15:33:45')", TYPE_INT, 1);
TestValue("timestamp_cmp('1966-05-04 15:33:45','1966-05-04 15:33:45')", TYPE_INT, 0);
TestValue("timestamp_cmp('1967-06-05','1966-05-04')", TYPE_INT, 1);
- TestValue("timestamp_cmp('15:33:45','16:34:45')", TYPE_INT, -1);
TestValue("timestamp_cmp('1966-05-04','1966-05-04 15:33:45')", TYPE_INT, -1);
TestIsNull("timestamp_cmp('','1966-05-04 15:33:45')", TYPE_INT);
@@ -6836,7 +6833,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestValue("int_months_between('1967-07-19','1967-07-19')", TYPE_INT, 0);
TestValue("int_months_between('2015-07-19','2015-08-18')", TYPE_INT, 0);
- TestIsNull("int_months_between('23:33:45','15:33:45')", TYPE_INT);
TestIsNull("int_months_between('','1966-06-04')", TYPE_INT);
TestValue("months_between('1967-07-19','1966-06-04')", TYPE_DOUBLE,
@@ -6847,7 +6843,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestValue("months_between('2015-02-28','2015-05-31')", TYPE_DOUBLE, -3);
TestValue("months_between('2012-02-29','2012-01-31')", TYPE_DOUBLE, 1);
- TestIsNull("months_between('23:33:45','15:33:45')", TYPE_DOUBLE);
TestIsNull("months_between('','1966-06-04')", TYPE_DOUBLE);
TestValue("datediff(cast('2011-12-22 09:10:11.12345678' as timestamp), \
@@ -6855,23 +6850,11 @@ TEST_P(ExprTest, TimestampFunctions) {
TestValue("datediff(cast('2012-12-22' as timestamp), \
cast('2011-12-22 09:10:11.12345678' as timestamp))", TYPE_INT, 366);
- TestIsNull("cast('24:59:59' as timestamp)", TYPE_TIMESTAMP);
+ TestIsNull("cast('2020-05-06 24:59:59' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast('10000-12-31' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast('10000-12-31 23:59:59' as timestamp)", TYPE_TIMESTAMP);
TestIsNull("cast('2000-12-31 24:59:59' as timestamp)", TYPE_TIMESTAMP);
- TestIsNull("year(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("quarter(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("month(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("dayofmonth(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("day(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("dayofyear(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("dayofweek(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("weekofyear(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("week(cast('09:10:11.000000' as timestamp))", TYPE_INT);
- TestIsNull("datediff(cast('09:10:11.12345678' as timestamp), "
- "cast('2012-12-22' as timestamp))", TYPE_INT);
-
TestIsNull("year(NULL)", TYPE_INT);
TestIsNull("quarter(NULL)", TYPE_INT);
TestIsNull("month(NULL)", TYPE_INT);
@@ -7199,9 +7182,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestIsNull("cast(trunc('2014-07-22 01:34:55 +0100', 'year') as STRING)", TYPE_STRING);
TestStringValue("cast(trunc(cast('2014-04-01' as timestamp), 'SYYYY') as string)",
"2014-01-01 00:00:00");
- TestIsNull("cast(trunc('01:34:55', 'year') as STRING)", TYPE_STRING);
- TestStringValue("cast(trunc(cast('07:02:03' as timestamp), 'MI') as string)",
- "07:02:00");
// note: no time value in string defaults to 00:00:00
TestStringValue("cast(trunc(cast('2014-01-01' as timestamp), 'MI') as string)",
"2014-01-01 00:00:00");
@@ -7399,7 +7379,7 @@ TEST_P(ExprTest, TimestampFunctions) {
"cast(trunc(cast('2012-09-10 07:59:03' as timestamp), 'MI') as string)",
"2012-09-10 07:59:00");
TestStringValue(
- "cast(trunc(cast('2012-09-10 07:59:59' as timestamp), 'MI') as string)",
+ "cast(trunc(cast('2012-09-10 07:59:59.123' as timestamp), 'MI') as string)",
"2012-09-10 07:59:00");
TestNonOkStatus(
"cast(trunc(cast('2012-09-10 07:59:59' as timestamp), 'MIN') as string)");
@@ -7497,8 +7477,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestStringValue(
"cast(cast('2012-01-01T09:10:11.12345-01:30' as timestamp) as string)",
"2012-01-01 09:10:11.123450000");
- TestStringValue("cast(cast('09:10:11+01:30' as timestamp) as string)", "09:10:11");
- TestStringValue("cast(cast('09:10:11-01:30' as timestamp) as string)", "09:10:11");
TestValue("unix_timestamp('2038-01-19T03:14:08-0100')", TYPE_BIGINT, 2147483648);
TestValue("unix_timestamp('2038/01/19T03:14:08+01:00', 'yyyy/MM/ddTHH:mm:ss')",
@@ -7511,8 +7489,6 @@ TEST_P(ExprTest, TimestampFunctions) {
TestStringValue("cast(trunc('2014-07-22T01:34:55+0100', 'year') as STRING)",
"2014-01-01 00:00:00");
- TestStringValue("cast(trunc(cast('07:02:03+01:30' as timestamp), 'MI') as string)",
- "07:02:00");
// Test timezone offset format
TestStringValue("from_unixtime(unix_timestamp('2012-01-01 19:10:11+02:30', \
@@ -7590,6 +7566,7 @@ TEST_P(ExprTest, TimestampFunctions) {
TestError("from_unixtime(unix_timestamp('10:00:00+0530 2010-01-01', \
'HH:mm:ss+hhmm yyyy-MM-dd'))");
+ TestError("to_timestamp('01:01:01', 'HH:mm:ss')");
TestError("to_timestamp('2012-02-28 11:10:11+0530', 'yyyy-MM-dd HH:mm:ss+hhdd')");
TestError("to_timestamp('2012-02-28+0530', 'yyyy-MM-dd+hhmm')");
TestError("to_timestamp('10:00:00+0530 2010-01-01', 'HH:mm:ss+hhmm yyyy-MM-dd')");
diff --git a/be/src/exprs/scalar-expr-evaluator.cc b/be/src/exprs/scalar-expr-evaluator.cc
index 14ec10e..e08d518 100644
--- a/be/src/exprs/scalar-expr-evaluator.cc
+++ b/be/src/exprs/scalar-expr-evaluator.cc
@@ -443,7 +443,7 @@ void ScalarExprEvaluator::InitBuiltinsDummy() {
MathFunctions::Pi(nullptr);
StringFunctions::Length(nullptr, StringVal::null());
TimestampFunctions::Year(nullptr, TimestampVal::null());
- TimestampFunctions::UnixAndFromUnixPrepare(nullptr, FunctionContext::FRAGMENT_LOCAL);
+ TimestampFunctions::FromUnixPrepare(nullptr, FunctionContext::FRAGMENT_LOCAL);
DateFunctions::Year(nullptr, DateVal::null());
UdfBuiltins::Pi(nullptr);
UtilityFunctions::Pid(nullptr);
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 5ffab1f..12e16c7 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -65,7 +65,7 @@ StringVal TimestampFunctions::StringValFromTimestamp(FunctionContext* context,
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx)){
+ if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx, FORMAT)){
ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
return StringVal::null();
}
@@ -165,11 +165,11 @@ TimestampVal TimestampFunctions::ToTimestamp(FunctionContext* context,
void* state = context->GetFunctionState(FunctionContext::THREAD_LOCAL);
DateTimeFormatContext* dt_ctx = reinterpret_cast<DateTimeFormatContext*>(state);
if (!context->IsArgConstant(1)) {
- dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
- if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx)) {
- ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
- return TimestampVal::null();
- }
+ dt_ctx->Reset(reinterpret_cast<const char*>(fmt.ptr), fmt.len);
+ if (!SimpleDateFormatTokenizer::Tokenize(dt_ctx, PARSE)) {
+ ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt, false);
+ return TimestampVal::null();
+ }
}
const TimestampValue& tv = TimestampValue::ParseSimpleDateFormat(
reinterpret_cast<const char*>(date.ptr), date.len, *dt_ctx);
diff --git a/be/src/exprs/timestamp-functions.cc b/be/src/exprs/timestamp-functions.cc
index 92c2d2e..cafc26e 100644
--- a/be/src/exprs/timestamp-functions.cc
+++ b/be/src/exprs/timestamp-functions.cc
@@ -151,8 +151,10 @@ TimestampVal TimestampFunctions::ToUtc(FunctionContext* context,
return ts_val_ret;
}
-void TimestampFunctions::UnixAndFromUnixPrepare(
- FunctionContext* context, FunctionContext::FunctionStateScope scope) {
+// The purpose of making this .cc only is to avoid moving the code of
+// CastDirection into the .h file
+void UnixAndFromUnixPrepare(FunctionContext* context,
+ FunctionContext::FunctionStateScope scope, CastDirection cast_mode) {
if (scope != FunctionContext::THREAD_LOCAL) return;
DateTimeFormatContext* dt_ctx = NULL;
if (context->IsArgConstant(1)) {
@@ -163,7 +165,7 @@ void TimestampFunctions::UnixAndFromUnixPrepare(
return;
}
dt_ctx = new DateTimeFormatContext(fmt_ref.ptr, fmt_ref.len);
- bool parse_result = SimpleDateFormatTokenizer::Tokenize(dt_ctx);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(dt_ctx, cast_mode);
if (!parse_result) {
delete dt_ctx;
ReportBadFormat(context, datetime_parse_util::GENERAL_ERROR, fmt_val, true);
@@ -189,4 +191,14 @@ void TimestampFunctions::UnixAndFromUnixClose(FunctionContext* context,
}
}
+void TimestampFunctions::ToUnixPrepare(
+ FunctionContext* context, FunctionContext::FunctionStateScope scope) {
+ UnixAndFromUnixPrepare(context, scope, PARSE);
+}
+
+void TimestampFunctions::FromUnixPrepare(
+ FunctionContext* context, FunctionContext::FunctionStateScope scope) {
+ UnixAndFromUnixPrepare(context, scope, FORMAT);
+}
+
}
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index 992ae34..602eeda 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -100,10 +100,12 @@ class TimestampFunctions {
static const std::map<std::string, int> DAYNAME_MAP;
/// Parse and initialize format string if it is a constant. Raise error if invalid.
- static void UnixAndFromUnixPrepare(FunctionContext* context,
- FunctionContext::FunctionStateScope scope);
static void UnixAndFromUnixClose(FunctionContext* context,
FunctionContext::FunctionStateScope scope);
+ static void ToUnixPrepare(FunctionContext* context,
+ FunctionContext::FunctionStateScope scope);
+ static void FromUnixPrepare(FunctionContext* context,
+ FunctionContext::FunctionStateScope scope);
/// Parses 'string_val' based on the format 'fmt'.
/// The time zone interpretation of the parsed timestamp is determined by
diff --git a/be/src/runtime/date-parse-util.cc b/be/src/runtime/date-parse-util.cc
index 1f820f5..9e94f28 100644
--- a/be/src/runtime/date-parse-util.cc
+++ b/be/src/runtime/date-parse-util.cc
@@ -108,7 +108,7 @@ bool DateParser::ParseSimpleDateFormat(const char* str, int len, bool accept_tim
// Generating context lazily as a fall back if default formats fail.
// ParseFormatTokenByStr() does not require a template format string.
DateTimeFormatContext lazy_ctx(str, trimmed_len);
- if (!SimpleDateFormatTokenizer::TokenizeByStr(&lazy_ctx, accept_time_toks, false)) {
+ if (!SimpleDateFormatTokenizer::TokenizeByStr(&lazy_ctx, accept_time_toks)) {
return IndicateDateParseFailure(date);
}
return ParseSimpleDateFormat(str, trimmed_len, lazy_ctx, date);
diff --git a/be/src/runtime/date-test.cc b/be/src/runtime/date-test.cc
index 3072dc5..18d65c0 100644
--- a/be/src/runtime/date-test.cc
+++ b/be/src/runtime/date-test.cc
@@ -154,7 +154,7 @@ void TestDateTokens(const vector<DateToken>& toks, int year, int month, int day,
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false)) << fmt_val;
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE, false)) << fmt_val;
DateValue dv = DateValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateDate(dv, year, month, day, fmt_val);
@@ -269,7 +269,7 @@ struct DateTC {
<< " expected date:" << expected_year << "/" << expected_month << "/"
<< expected_day;
- bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE, false);
if (fmt_should_fail) {
EXPECT_FALSE(parse_result) << desc.str();
return;
@@ -498,7 +498,7 @@ struct DateFormatTC {
desc << "DateFormatTC [" << id << "]: " << "days_since_epoch:" << days_since_epoch
<< " fmt:" << fmt << " str:" << str;
- ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, false)) << desc.str();
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE, false)) << desc.str();
const DateValue cust_dv(days_since_epoch);
EXPECT_TRUE(cust_dv.IsValid()) << desc.str();
diff --git a/be/src/runtime/datetime-iso-sql-format-tokenizer.cc b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
index a7eaef3..4c24960 100644
--- a/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
+++ b/be/src/runtime/datetime-iso-sql-format-tokenizer.cc
@@ -200,6 +200,9 @@ FormatTokenizationResult IsoSqlFormatTokenizer::CheckIncompatibilities() const {
return SUCCESS;
}
+ DCHECK(cast_mode_ == PARSE);
+ if(!dt_ctx_->has_date_toks) return NO_DATE_TOKENS_ERROR;
+
short provided_year_count = IsUsedToken("YYYY") + IsUsedToken("YYY") +
IsUsedToken("YY") + IsUsedToken("Y");
short provided_round_year_count = IsUsedToken("RRRR") + IsUsedToken("RR");
diff --git a/be/src/runtime/datetime-parser-common.cc b/be/src/runtime/datetime-parser-common.cc
index 4186fe7..b931f3d 100644
--- a/be/src/runtime/datetime-parser-common.cc
+++ b/be/src/runtime/datetime-parser-common.cc
@@ -125,6 +125,9 @@ void ReportBadFormat(FunctionContext* context, FormatTokenizationResult error_ty
case NO_DATETIME_TOKENS_ERROR:
ss << "PARSE ERROR: No datetime tokens provided.";
break;
+ case NO_DATE_TOKENS_ERROR:
+ ss << "PARSE ERROR: No date tokens provided.";
+ break;
case MISPLACED_FX_MODIFIER_ERROR:
ss << "PARSE ERROR: FX modifier should be at the beginning of the format string.";
case CONFLICTING_DAY_OF_WEEK_TOKENS_ERROR:
diff --git a/be/src/runtime/datetime-parser-common.h b/be/src/runtime/datetime-parser-common.h
index ddac490..6ae0a92 100644
--- a/be/src/runtime/datetime-parser-common.h
+++ b/be/src/runtime/datetime-parser-common.h
@@ -122,6 +122,7 @@ enum FormatTokenizationResult {
DATE_WITH_TIME_ERROR,
CONFLICTING_FRACTIONAL_SECOND_TOKENS_ERROR,
TEXT_TOKEN_NOT_CLOSED,
+ NO_DATE_TOKENS_ERROR,
NO_DATETIME_TOKENS_ERROR,
MISPLACED_FX_MODIFIER_ERROR,
QUARTER_NOT_ALLOWED_FOR_PARSING,
diff --git a/be/src/runtime/datetime-simple-date-format-parser.cc b/be/src/runtime/datetime-simple-date-format-parser.cc
index 2505fcf..33ca531 100644
--- a/be/src/runtime/datetime-simple-date-format-parser.cc
+++ b/be/src/runtime/datetime-simple-date-format-parser.cc
@@ -35,18 +35,14 @@ namespace datetime_parse_util {
bool SimpleDateFormatTokenizer::initialized = false;
const int SimpleDateFormatTokenizer::DEFAULT_DATE_FMT_LEN = 10;
-const int SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN = 8;
-const int SimpleDateFormatTokenizer::DEFAULT_TIME_FRAC_FMT_LEN = 18;
const int SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_FMT_LEN = 19;
const int SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_FMT_LEN = 29;
DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_CTX;
DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_SHORT_ISO_DATE_TIME_CTX;
DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_DATE_CTX;
-DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_TIME_CTX;
DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_DATE_TIME_CTX[10];
DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_ISO_DATE_TIME_CTX[10];
-DateTimeFormatContext SimpleDateFormatTokenizer::DEFAULT_TIME_FRAC_CTX[10];
void SimpleDateFormatTokenizer::InitCtx() {
if (initialized) return;
@@ -57,38 +53,27 @@ void SimpleDateFormatTokenizer::InitCtx() {
for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
DEFAULT_DATE_TIME_CTX[i].Reset(DATE_TIME_CTX_FMT,
DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- Tokenize(&DEFAULT_DATE_TIME_CTX[i]);
+ Tokenize(&DEFAULT_DATE_TIME_CTX[i], PARSE);
}
// Setup the default ISO date/time context yyyy-MM-ddTHH:mm:ss.SSSSSSSSS
for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
DEFAULT_ISO_DATE_TIME_CTX[i].Reset("yyyy-MM-ddTHH:mm:ss.SSSSSSSSS",
DEFAULT_DATE_TIME_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- Tokenize(&DEFAULT_ISO_DATE_TIME_CTX[i]);
+ Tokenize(&DEFAULT_ISO_DATE_TIME_CTX[i], PARSE);
}
// Setup the short default date/time context yyyy-MM-dd HH:mm:ss
DEFAULT_SHORT_DATE_TIME_CTX.Reset("yyyy-MM-dd HH:mm:ss");
- Tokenize(&DEFAULT_SHORT_DATE_TIME_CTX);
+ Tokenize(&DEFAULT_SHORT_DATE_TIME_CTX, PARSE);
// Setup the short default ISO date/time context yyyy-MM-ddTHH:mm:ss
DEFAULT_SHORT_ISO_DATE_TIME_CTX.Reset("yyyy-MM-ddTHH:mm:ss");
- Tokenize(&DEFAULT_SHORT_ISO_DATE_TIME_CTX);
+ Tokenize(&DEFAULT_SHORT_ISO_DATE_TIME_CTX, PARSE);
// Setup the default short date context yyyy-MM-dd
DEFAULT_DATE_CTX.Reset("yyyy-MM-dd");
- Tokenize(&DEFAULT_DATE_CTX);
-
- // Setup the default short time context HH:mm:ss
- DEFAULT_TIME_CTX.Reset("HH:mm:ss");
- Tokenize(&DEFAULT_TIME_CTX);
-
- // Setup the default short time context with fractional seconds HH:mm:ss.SSSSSSSSS
- for (int i = FRACTIONAL_MAX_LEN; i >= 0; --i) {
- DEFAULT_TIME_FRAC_CTX[i].Reset(DATE_TIME_CTX_FMT + 11,
- DEFAULT_TIME_FRAC_FMT_LEN - (FRACTIONAL_MAX_LEN - i));
- Tokenize(&DEFAULT_TIME_FRAC_CTX[i]);
- }
+ Tokenize(&DEFAULT_DATE_CTX, PARSE);
// Flag that the parser is ready.
initialized = true;
@@ -112,8 +97,8 @@ bool SimpleDateFormatTokenizer::IsValidTZOffset(const char* str_begin,
return false;
}
-bool SimpleDateFormatTokenizer::Tokenize(DateTimeFormatContext* dt_ctx,
- bool accept_time_toks) {
+bool SimpleDateFormatTokenizer::Tokenize(
+ DateTimeFormatContext* dt_ctx, CastDirection cast_mode, bool accept_time_toks) {
DCHECK(dt_ctx != NULL);
DCHECK(dt_ctx->fmt != NULL);
DCHECK(dt_ctx->fmt_len > 0);
@@ -191,7 +176,8 @@ bool SimpleDateFormatTokenizer::Tokenize(DateTimeFormatContext* dt_ctx,
str += tok.len;
dt_ctx->toks.push_back(tok);
}
- return dt_ctx->has_date_toks || dt_ctx->has_time_toks;
+ if (cast_mode == PARSE) return (dt_ctx->has_date_toks);
+ return (dt_ctx->has_date_toks || dt_ctx->has_time_toks);
}
const char* SimpleDateFormatTokenizer::ParseDigitToken(const char* str,
@@ -215,7 +201,7 @@ const char* SimpleDateFormatTokenizer::ParseSeparatorToken(const char* str,
}
bool SimpleDateFormatTokenizer::TokenizeByStr( DateTimeFormatContext* dt_ctx,
- bool accept_time_toks, bool accept_time_toks_only) {
+ bool accept_time_toks) {
DCHECK(dt_ctx != NULL);
DCHECK(dt_ctx->fmt != NULL);
DCHECK_GT(dt_ctx->fmt_len, 0);
@@ -286,7 +272,7 @@ bool SimpleDateFormatTokenizer::TokenizeByStr( DateTimeFormatContext* dt_ctx,
if (!accept_time_toks) return false;
// If no date tokens were found and time tokens on their own are not allowed, return
// false.
- if (!dt_ctx->has_date_toks && !accept_time_toks_only) return false;
+ if (!dt_ctx->has_date_toks) return false;
// Parse the 1 or 2 digit hour
if (tok_end - str != 1 && tok_end - str != 2) return false;
@@ -355,72 +341,56 @@ const DateTimeFormatContext* SimpleDateFormatTokenizer::GetDefaultFormatContext(
DCHECK(str != nullptr);
DCHECK(len > 0);
- if (LIKELY(len >= DEFAULT_TIME_FMT_LEN)) {
- // Check if this string starts with a date component
- if (str[4] == '-' && str[7] == '-') {
- // Do we have a date component only?
- if (len == DEFAULT_DATE_FMT_LEN) {
- return &DEFAULT_DATE_CTX;
- }
+ // Check if this string starts with a date component
+ if (str[4] == '-' && str[7] == '-') {
+ // Do we have a date component only?
+ if (len == DEFAULT_DATE_FMT_LEN) {
+ return &DEFAULT_DATE_CTX;
+ }
- // We have a time component as well. Do we accept it?
- if (!accept_time_toks) return nullptr;
-
- switch (len) {
- case DEFAULT_SHORT_DATE_TIME_FMT_LEN: {
- if (LIKELY(str[13] == ':')) {
- switch (str[10]) {
- case ' ':
- return &DEFAULT_SHORT_DATE_TIME_CTX;
- case 'T':
- return &DEFAULT_SHORT_ISO_DATE_TIME_CTX;
- }
+ // We have a time component as well. Do we accept it?
+ if (!accept_time_toks) return nullptr;
+
+ switch (len) {
+ case DEFAULT_SHORT_DATE_TIME_FMT_LEN: {
+ if (LIKELY(str[13] == ':')) {
+ switch (str[10]) {
+ case ' ':
+ return &DEFAULT_SHORT_DATE_TIME_CTX;
+ case 'T':
+ return &DEFAULT_SHORT_ISO_DATE_TIME_CTX;
}
- break;
}
- case DEFAULT_DATE_TIME_FMT_LEN: {
- if (LIKELY(str[13] == ':')) {
- switch (str[10]) {
- case ' ':
- return &DEFAULT_DATE_TIME_CTX[9];
- case 'T':
- return &DEFAULT_ISO_DATE_TIME_CTX[9];
- }
+ break;
+ }
+ case DEFAULT_DATE_TIME_FMT_LEN: {
+ if (LIKELY(str[13] == ':')) {
+ switch (str[10]) {
+ case ' ':
+ return &DEFAULT_DATE_TIME_CTX[9];
+ case 'T':
+ return &DEFAULT_ISO_DATE_TIME_CTX[9];
}
- break;
}
- default: {
- // There is likely a fractional component that's below the expected 9 chars.
- // We will need to work out which default context to use that corresponds to
- // the fractional length in the string.
- if (LIKELY(len > DEFAULT_SHORT_DATE_TIME_FMT_LEN)
- && LIKELY(str[19] == '.') && LIKELY(str[13] == ':')) {
- switch (str[10]) {
- case ' ': {
- return &DEFAULT_DATE_TIME_CTX[len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
- }
- case 'T': {
- return &DEFAULT_ISO_DATE_TIME_CTX
- [len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
- }
+ break;
+ }
+ default: {
+ // There is likely a fractional component that's below the expected 9 chars.
+ // We will need to work out which default context to use that corresponds to
+ // the fractional length in the string.
+ if (LIKELY(len > DEFAULT_SHORT_DATE_TIME_FMT_LEN)
+ && LIKELY(str[19] == '.') && LIKELY(str[13] == ':')) {
+ switch (str[10]) {
+ case ' ': {
+ return &DEFAULT_DATE_TIME_CTX[len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
+ }
+ case 'T': {
+ return &DEFAULT_ISO_DATE_TIME_CTX
+ [len - DEFAULT_SHORT_DATE_TIME_FMT_LEN - 1];
}
}
- break;
- }
- }
- } else {
- // 'str' string does not start with a date component.
- // Do we accept time component only?
- if (!accept_time_toks || !accept_time_toks_only) return nullptr;
-
- // Parse time component.
- if (str[2] == ':' && str[5] == ':' && isdigit(str[7])) {
- len = min(len, DEFAULT_TIME_FRAC_FMT_LEN);
- if (len > DEFAULT_TIME_FMT_LEN && str[8] == '.') {
- return &DEFAULT_TIME_FRAC_CTX[len - DEFAULT_TIME_FMT_LEN - 1];
- } else {
- return &DEFAULT_TIME_CTX;
}
+ break;
}
}
}
diff --git a/be/src/runtime/datetime-simple-date-format-parser.h b/be/src/runtime/datetime-simple-date-format-parser.h
index 6557b6e..58b233b 100644
--- a/be/src/runtime/datetime-simple-date-format-parser.h
+++ b/be/src/runtime/datetime-simple-date-format-parser.h
@@ -74,8 +74,6 @@ class SimpleDateFormatTokenizer {
public:
/// Constants to hold default format lengths.
static const int DEFAULT_DATE_FMT_LEN;
- static const int DEFAULT_TIME_FMT_LEN;
- static const int DEFAULT_TIME_FRAC_FMT_LEN;
static const int DEFAULT_SHORT_DATE_TIME_FMT_LEN;
static const int DEFAULT_DATE_TIME_FMT_LEN;
@@ -83,8 +81,9 @@ public:
/// dt_ctx -- output date/time format context
/// accept_time_toks -- if true, time tokens are accepted. Otherwise time tokens are
/// rejected.
+ /// cast_mode -- indicates if it is a 'datetime to string' or 'string to datetime' cast
/// Return true if the parse was successful.
- static bool Tokenize(DateTimeFormatContext* dt_ctx,
+ static bool Tokenize(DateTimeFormatContext* dt_ctx, CastDirection cast_mode,
bool accept_time_toks = true);
/// Parse the date/time string to generate the DateTimeFormatToken required by
@@ -95,11 +94,10 @@ public:
/// dt_ctx -- date/time format context (must contain valid tokens)
/// accept_time_toks -- if true, time tokens are accepted, otherwise time tokens are
/// rejected.
- /// accept_time_toks_only -- if true, time tokens w/o date tokens are accepted.
/// Otherwise, they are rejected.
/// Return true if the date/time was successfully parsed.
static bool TokenizeByStr(DateTimeFormatContext* dt_ctx,
- bool accept_time_toks = true, bool accept_time_toks_only = true);
+ bool accept_time_toks = true);
/// Parse date/time string to find the corresponding default date/time format context.
/// The string must adhere to a default date/time format.
@@ -127,10 +125,8 @@ private:
static DateTimeFormatContext DEFAULT_SHORT_DATE_TIME_CTX;
static DateTimeFormatContext DEFAULT_SHORT_ISO_DATE_TIME_CTX;
static DateTimeFormatContext DEFAULT_DATE_CTX;
- static DateTimeFormatContext DEFAULT_TIME_CTX;
static DateTimeFormatContext DEFAULT_DATE_TIME_CTX[10];
static DateTimeFormatContext DEFAULT_ISO_DATE_TIME_CTX[10];
- static DateTimeFormatContext DEFAULT_TIME_FRAC_CTX[10];
/// Checks if str_begin point to the beginning of a valid timezone offset.
static bool IsValidTZOffset(const char* str_begin, const char* str_end);
diff --git a/be/src/runtime/timestamp-parse-util.cc b/be/src/runtime/timestamp-parse-util.cc
index 63340ea..aa72598 100644
--- a/be/src/runtime/timestamp-parse-util.cc
+++ b/be/src/runtime/timestamp-parse-util.cc
@@ -85,10 +85,10 @@ bool TimestampParser::ParseSimpleDateFormat(const char* str, int len,
// Strip if there is a 'Z' suffix
if (trimmed_len > 0 && str[trimmed_len - 1] == 'Z') {
--trimmed_len;
- } else if (trimmed_len > SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN &&
- (str[4] == '-' || str[2] == ':')) {
+ } else if (trimmed_len > SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_FMT_LEN
+ && str[4] == '-') {
// Strip timezone offset if it seems like a valid timestamp string.
- int curr_pos = SimpleDateFormatTokenizer::DEFAULT_TIME_FMT_LEN;
+ int curr_pos = SimpleDateFormatTokenizer::DEFAULT_SHORT_DATE_TIME_FMT_LEN;
// Timezone offset will be at least two bytes long, no need to check last
// two bytes.
while (curr_pos < trimmed_len - 2) {
diff --git a/be/src/runtime/timestamp-test.cc b/be/src/runtime/timestamp-test.cc
index 8539695..813380f 100644
--- a/be/src/runtime/timestamp-test.cc
+++ b/be/src/runtime/timestamp-test.cc
@@ -184,7 +184,7 @@ void TestTimestampTokens(vector<TimestampToken>* toks, int year, int month,
}
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << fmt_val;
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE)) << fmt_val;
TimestampValue tv =
TimestampValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateTimestamp(tv, fmt, val, fmt_val, year, month, day, hours, mins, secs,
@@ -212,7 +212,7 @@ void TestTimestampTokens(vector<TimestampToken>* toks, int year, int month,
}
string fmt_val = "Format: " + fmt + ", Val: " + val;
DateTimeFormatContext dt_ctx(fmt.c_str());
- ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << fmt_val;
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE)) << fmt_val;
TimestampValue tv =
TimestampValue::ParseSimpleDateFormat(val.c_str(), val.length(), dt_ctx);
ValidateTimestamp(tv, fmt, val, fmt_val, year, month, day, hours, mins, secs,
@@ -399,24 +399,9 @@ TEST(TimestampTest, Basic) {
EXPECT_EQ(dv1.date().month(), 1);
EXPECT_EQ(dv1.date().day(), 20);
- char t1[] = "10:11:12.123456789";
- char t2[] = "00:00:00";
- TimestampValue tv1 = TimestampValue::ParseSimpleDateFormat(t1, strlen(t1));
- TimestampValue tv2 = TimestampValue::ParseSimpleDateFormat(t2, strlen(t2));
-
- EXPECT_NE(tv1, tv2);
- EXPECT_NE(tv1, v2);
-
- EXPECT_EQ(tv1.time().hours(), 10);
- EXPECT_EQ(tv1.time().minutes(), 11);
- EXPECT_EQ(tv1.time().seconds(), 12);
- EXPECT_EQ(tv1.time().fractional_seconds(), 123456789);
- EXPECT_EQ(tv2.time().fractional_seconds(), 0);
-
// Test variable fraction lengths
const char* FRACTION_MAX_STR = "123456789";
- const char* TEST_VALS[] = { "2013-12-10 12:04:17.", "2013-12-10T12:04:17.",
- "12:04:17." };
+ const char* TEST_VALS[] = { "2013-12-10 12:04:17.", "2013-12-10T12:04:17."};
const int TEST_VAL_CNT = sizeof(TEST_VALS) / sizeof(char*);
for (int i = 0; i < TEST_VAL_CNT; ++i) {
const int VAL_LEN = strlen(TEST_VALS[i]);
@@ -572,13 +557,13 @@ TEST(TimestampTest, Basic) {
// Test out of range day
(TimestampTC("yyyyMMdd", "20130100", false, true))
// Test out of range hour
- (TimestampTC("HH:mm:ss", "24:01:01", false, true))
+ (TimestampTC("yyyy-MM-dd HH:mm:ss", "1400-01-01 24:01:01", false, true))
// Test out of range minute
- (TimestampTC("HH:mm:ss", "23:60:01", false, true))
+ (TimestampTC("yyyy-MM-dd HH:mm:ss", "1400-01-01 23:60:01", false, true))
// Test out of range second
- (TimestampTC("HH:mm:ss", "23:01:60", false, true))
+ (TimestampTC("yyyy-MM-dd HH:mm:ss", "1400-01-01 23:01:60", false, true))
// Test characters where numbers should be
- (TimestampTC("HH:mm:ss", "aa:01:01", false, true))
+ (TimestampTC("yyyy-MM-dd HH:mm:ss", "1400-01-01 aa:01:01", false, true))
// Test missing year
(TimestampTC("MMdd", "1201", false, true))
// Test missing month
@@ -619,19 +604,27 @@ TEST(TimestampTest, Basic) {
(TimestampTC("y-M-d", "2013-11-13", false, true, false, 2013, 11, 13))
(TimestampTC("y-M-d", "13-1-3", false, true, false, 1913, 1, 3))
// Test short hour token
- (TimestampTC("H:mm:ss", "14:24:34", false, false, true, 0, 0, 0, 14, 24, 34))
- (TimestampTC("H:mm:ss", "4:24:34", false, false, true, 0, 0, 0, 4, 24, 34))
+ (TimestampTC("yyyy-MM-dd H:mm:ss", "2020-05-11 14:24:34", false, true, true, 2020,
+ 05, 11, 14, 24, 34))
+ (TimestampTC("yyyy-MM-dd H:mm:ss", "2020-05-11 4:24:34", false, true, true, 2020,
+ 05, 11, 4, 24, 34))
// Test short minute token
- (TimestampTC("HH:m:ss", "14:24:34", false, false, true, 0, 0, 0, 14, 24, 34))
- (TimestampTC("HH:m:ss", "1:24:34", false, true))
+ (TimestampTC("yyyy-MM-dd HH:m:ss", "2020-05-11 14:24:34", false, true, true, 2020,
+ 05, 11, 14, 24, 34))
+ (TimestampTC("yyyy-MM-dd HH:m:ss", "2020-05-11 1:24:34", false, true))
// Test short second token
- (TimestampTC("HH:mm:s", "14:24:34", false, false, true, 0, 0, 0, 14, 24, 34))
+ (TimestampTC("yyyy-MM-dd HH:mm:s", "2020-05-11 14:24:34", false, true, true, 2020,
+ 05, 11, 14, 24, 34))
+ (TimestampTC("yyyy-MM-dd HH:mm:s", "2020-05-11 14:24:3", false, true, true, 2020,
+ 05, 11, 14, 24, 3))
// Test short all time tokens
- (TimestampTC("H:m:s", "11:22:33", false, false, true, 0, 0, 0, 11, 22, 33))
- (TimestampTC("H:m:s", "1:2:3", false, false, true, 0, 0, 0, 1, 2, 3))
+ (TimestampTC("yyyy-MM-dd H:m:s", "2020-05-11 11:22:33", false, true, true, 2020,
+ 05, 11, 11, 22, 33))
+ (TimestampTC("yyyy-MM-dd H:m:s", "2020-05-11 1:2:3", false, true, true, 2020, 05,
+ 11, 1, 2, 3))
// Test short fraction token
- (TimestampTC("HH:mm:ss:S", "14:24:34:1234", false, false, true, 0, 0, 0, 14, 24, 34,
- 123400000));
+ (TimestampTC("yyyy-MM-dd HH:mm:ss:S", "2020-05-11 14:24:34:1234", false, true,
+ true, 2020, 05, 11, 14, 24, 34, 123400000));
// Loop through custom parse/format test cases and execute each one. Each test case
// will be explicitly set with a pass/fail expectation related to either the format
// or literal value.
@@ -639,7 +632,7 @@ TEST(TimestampTest, Basic) {
TimestampTC test_case = test_cases[i];
DateTimeFormatContext dt_ctx(test_case.fmt);
dt_ctx.SetCenturyBreakAndCurrentTime(now);
- bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx);
+ bool parse_result = SimpleDateFormatTokenizer::Tokenize(&dt_ctx, PARSE);
if (test_case.fmt_should_fail) {
EXPECT_FALSE(parse_result) << "TC: " << i;
continue;
@@ -689,11 +682,11 @@ TEST(TimestampTest, Basic) {
vector<TimestampFormatTC> fmt_test_cases = list_of
(TimestampFormatTC(1382337792, "yyyy-MM-dd HH:mm:ss.SSSSSSSSS",
"2013-10-21 06:43:12.000000000"))
- // Test just formatting time tokens
+ // Test formatting only time tokens
(TimestampFormatTC(1382337792, "HH:mm:ss.SSSSSSSSS", "06:43:12.000000000"))
(TimestampFormatTC(0, "yyyy-MM-ddTHH:mm:SS.SSSSSSSSSZ",
"1970-01-01T00:00:00.000000000Z"))
- // Test just formatting date tokens
+ // Test formatting only date tokens
(TimestampFormatTC(965779200, "yyyy-MM-dd", "2000-08-09"))
// Test short form date tokens
(TimestampFormatTC(965779200, "yyyy-M-d", "2000-8-9"))
@@ -705,13 +698,14 @@ TEST(TimestampTest, Basic) {
(TimestampFormatTC(965779200, "dddddd/dd/d", "000009/09/9"))
// Test padding on double digits
(TimestampFormatTC(1382337792, "dddddd/dd/dd", "000021/21/21"))
- // Test just formatting time tokens on a ts value generated from a date
+ // Test formatting only time tokens on a ts value generated from a date
(TimestampFormatTC(965779200, "HH:mm:ss", "00:00:00"));
// Loop through format test cases
for (int i = 0; i < fmt_test_cases.size(); ++i) {
TimestampFormatTC test_case = fmt_test_cases[i];
DateTimeFormatContext dt_ctx(test_case.fmt);
- ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx)) << "TC: " << i;
+ ASSERT_TRUE(SimpleDateFormatTokenizer::Tokenize(&dt_ctx, FORMAT))
+ << "TC: " << i;
TimestampValue cust_tv = TimestampValue::FromUnixTime(test_case.ts, UTCPTR);
EXPECT_NE(cust_tv.date(), not_a_date) << "TC: " << i;
EXPECT_NE(cust_tv.time(), not_a_date_time) << "TC: " << i;
@@ -1065,4 +1059,3 @@ TEST(TimestampTest, TimezoneConversions) {
}
}
}
-
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index 0863518..279940f 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -152,7 +152,7 @@ class TimestampValue {
/// Returns a TimestampVal representation in the output variable.
/// Returns null if the TimestampValue instance doesn't have a valid date or time.
void ToTimestampVal(impala_udf::TimestampVal* tv) const {
- if (!HasDateOrTime()) {
+ if (!HasDate()) {
tv->is_null = true;
return;
}
@@ -181,7 +181,6 @@ class TimestampValue {
bool HasDate() const { return !date_.is_special(); }
bool HasTime() const { return !time_.is_special(); }
- bool HasDateOrTime() const { return HasDate() || HasTime(); }
bool HasDateAndTime() const { return HasDate() && HasTime(); }
std::string ToString() const;
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 7501932..362d100 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -144,6 +144,7 @@ testdata/data/decimal_rtf_tbl.txt
testdata/data/overflow.txt
testdata/data/text-comma-backslash-newline.txt
testdata/data/text-dollar-hash-pipe.txt
+testdata/data/dateless_timestamps.txt
testdata/data/text_large_zstd.txt
testdata/data/text_large_zstd.zst
testdata/data/widerow.txt
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index 18c86d0..c366552 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -226,7 +226,7 @@ visible_functions = [
[['unix_timestamp'], 'BIGINT', [], '_ZN6impala18TimestampFunctions4UnixEPN10impala_udf15FunctionContextE'],
[['unix_timestamp'], 'BIGINT', ['TIMESTAMP'], '_ZN6impala18TimestampFunctions4UnixEPN10impala_udf15FunctionContextERKNS1_12TimestampValE'],
[['unix_timestamp'], 'BIGINT', ['STRING', 'STRING'], '_ZN6impala18TimestampFunctions4UnixEPN10impala_udf15FunctionContextERKNS1_9StringValES6_',
- '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
+ '_ZN6impala18TimestampFunctions13ToUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
[['unix_micros_to_utc_timestamp'], 'TIMESTAMP', ['BIGINT'],
'_ZN6impala18TimestampFunctions24UnixMicrosToUtcTimestampEPN10impala_udf15FunctionContextERKNS1_9BigIntValE'],
@@ -236,13 +236,13 @@ visible_functions = [
'_ZN6impala18TimestampFunctions8FromUnixIN10impala_udf6IntValEEENS2_9StringValEPNS2_15FunctionContextERKT_'],
[['from_unixtime'], 'STRING', ['INT', 'STRING'],
'_ZN6impala18TimestampFunctions8FromUnixIN10impala_udf6IntValEEENS2_9StringValEPNS2_15FunctionContextERKT_RKS4_',
- '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
+ '_ZN6impala18TimestampFunctions15FromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
[['from_unixtime'], 'STRING', ['BIGINT'],
'_ZN6impala18TimestampFunctions8FromUnixIN10impala_udf9BigIntValEEENS2_9StringValEPNS2_15FunctionContextERKT_'],
[['from_unixtime'], 'STRING', ['BIGINT', 'STRING'],
'_ZN6impala18TimestampFunctions8FromUnixIN10impala_udf9BigIntValEEENS2_9StringValEPNS2_15FunctionContextERKT_RKS4_',
- '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
+ '_ZN6impala18TimestampFunctions15FromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
[['now', 'current_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions3NowEPN10impala_udf15FunctionContextE'],
[['utc_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions12UtcTimestampEPN10impala_udf15FunctionContextE'],
@@ -261,11 +261,11 @@ visible_functions = [
'_ZN6impala18TimestampFunctions11ToTimestampEPN10impala_udf15FunctionContextERKNS1_9BigIntValE'],
[['to_timestamp'], 'TIMESTAMP', ['STRING', 'STRING'],
'_ZN6impala18TimestampFunctions11ToTimestampEPN10impala_udf15FunctionContextERKNS1_9StringValES6_',
- '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
+ '_ZN6impala18TimestampFunctions13ToUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
[['from_timestamp'], 'STRING', ['TIMESTAMP', 'STRING'],
'impala::TimestampFunctions::FromTimestamp',
- '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
+ '_ZN6impala18TimestampFunctions15FromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
'_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
# Date functions
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
index 1552ceb..40334b1 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
@@ -480,8 +480,7 @@ public class AnalyzeKuduDDLTest extends FrontendTestBase {
AnalysisError("create table tdefault (id int primary key, " +
"ts timestamp not null default cast('00:00:00' as timestamp)) " +
"partition by hash(id) partitions 3 stored as kudu",
- "CAST('00:00:00' AS TIMESTAMP) cannot be cast to a TIMESTAMP literal.",
- isExternalPurgeTbl);
+ "Default value of NULL not allowed on non-nullable column:", isExternalPurgeTbl);
AnalysisError("create table tdefault (id int primary key, " +
"ts timestamp not null default '2009-1 foo') " +
"partition by hash(id) partitions 3 stored as kudu",
diff --git a/testdata/data/README b/testdata/data/README
index 9469d56..0adac6c 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -532,3 +532,15 @@ alltypes_non_acid.orc:
Non-acid ORC file generated by Hive 3.1 with the following command:
CREATE TABLE alltypes_clone STORED AS ORC AS SELECT * FROM functional.alltypes.
It's used as an original file in ACID tests.
+
+dateless_timestamps.parq:
+Parquet file generated before the removal of dateless timestamp support.
+Created as: CREATE TABLE timestamps_pq(t TIMESTAMP) STORED AS parquet;
+It contains the folloving lines:
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00
+1996-04-22 10:00:00
+1996-04-22 00:00:00
+10:00:00.432100000
+10:00:00
diff --git a/testdata/data/dateless_timestamps.parq b/testdata/data/dateless_timestamps.parq
new file mode 100644
index 0000000..d2d9250
Binary files /dev/null and b/testdata/data/dateless_timestamps.parq differ
diff --git a/testdata/data/dateless_timestamps.txt b/testdata/data/dateless_timestamps.txt
new file mode 100644
index 0000000..9b10a62
--- /dev/null
+++ b/testdata/data/dateless_timestamps.txt
@@ -0,0 +1,7 @@
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00
+1996-04-22 10:00:00
+1996-04-22 00:00:00
+10:00:00.432100000
+10:00:00
diff --git a/testdata/data/lazy_timestamp.csv b/testdata/data/lazy_timestamp.csv
index d44db1d..5447b57 100644
--- a/testdata/data/lazy_timestamp.csv
+++ b/testdata/data/lazy_timestamp.csv
@@ -1,13 +1,6 @@
2001-1-2
2001-1-02
2001-01-2
-1:6:8
-01:6:8
-1:06:8
-1:6:08
-1:6:8.123456789101112
-1:6:8.123456789
-1:6:8.12345
2001-1-2 1:6:8
2001-1-2 1:6:8.123456
2001-1-2 1:6:8.123456789101112
diff --git a/testdata/workloads/functional-query/queries/QueryTest/date.test b/testdata/workloads/functional-query/queries/QueryTest/date.test
index e9a0729..d6b47fb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/date.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/date.test
@@ -597,13 +597,6 @@ select cast('not-a-date' as date);
UDF ERROR: String to Date parse failed. Invalid string val: "not-a-date"
====
---- QUERY
-# TIMESTAMP->DATE conversion fails because timestamp has no date component. Note that
-# time-only timestamps are technically valid but impractical.
-select cast(cast('23:59:59' as timestamp) as date);
----- CATCH
-UDF ERROR: Timestamp to Date conversion failed. Timestamp has no date component.
-====
----- QUERY
# Basic test for FORMAT clause in CAST().
select cast("2014-11-01" as date format "YYYY-MM-DD");
---- RESULTS
diff --git a/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_parquet.test b/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_parquet.test
new file mode 100644
index 0000000..cfaa09e
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_parquet.test
@@ -0,0 +1,25 @@
+====
+---- QUERY
+SET abort_on_error=1;
+SELECT * FROM $DATABASE.timestamp_table;
+---- TYPES
+TIMESTAMP
+---- CATCH
+timestamp_table/dateless_timestamps.parq' column 't' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31.
+====
+---- QUERY
+SET abort_on_error=0;
+SELECT * FROM $DATABASE.timestamp_table;
+---- TYPES
+TIMESTAMP
+---- RESULTS
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00
+1996-04-22 10:00:00
+1996-04-22 00:00:00
+NULL
+NULL
+---- ERRORS
+Parquet file '__HDFS_FILENAME__' column 't' contains an out of range timestamp. The valid date range is 1400-01-01..9999-12-31. (1 of 2 similar)
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_text.test b/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_text.test
new file mode 100644
index 0000000..afb7109
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/dateless_timestamp_text.test
@@ -0,0 +1,29 @@
+====
+---- QUERY
+SET abort_on_error=1;
+SELECT * FROM $DATABASE.timestamp_text_table;
+---- TYPES
+TIMESTAMP
+---- CATCH
+Error converting column: 0 to TIMESTAMP
+Error parsing row: file: $NAMENODE/test-warehouse/test_dateless_timestamp_text_
+====
+---- QUERY
+SET abort_on_error=0;
+SELECT * FROM $DATABASE.timestamp_text_table;
+---- TYPES
+TIMESTAMP
+---- RESULTS
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00.432100000
+1996-04-22 10:00:00
+1996-04-22 10:00:00
+1996-04-22 00:00:00
+NULL
+NULL
+---- ERRORS
+Error converting column: 0 to TIMESTAMP
+Error converting column: 0 to TIMESTAMP
+Error parsing row: file: __HDFS_FILENAME__, before offset: 148
+Error parsing row: file: __HDFS_FILENAME__, before offset: 148
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 03fb03b..e013d87 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -2966,18 +2966,26 @@ select cast('2001-1-2 1:5:3.123' as timestamp)
timestamp
====
---- QUERY
-select cast('1:5:3' as timestamp)
+select cast('01:05:03' as timestamp)
---- RESULTS
-01:05:03
+NULL
---- TYPES
timestamp
====
---- QUERY
-select cast('1:5:3.1234567' as timestamp)
+select to_timestamp('01:01:01', 'HH:mm:ss');
+---- CATCH
+Query aborted:Bad date/time conversion format: HH:mm:ss
+====
+---- QUERY
+select to_timestamp('01:01:01.123', 'HH:mm:ss.SSS');
+---- CATCH
+Query aborted:Bad date/time conversion format: HH:mm:ss
+====
+---- QUERY
+select from_unixtime(1382337792,"HH:mm:ss.SSSSSSSSS");
---- RESULTS
-01:05:03.123456700
----- TYPES
-timestamp
+'06:43:12.000000000'
====
---- QUERY
select cast('2001-1-2' as timestamp)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/select-lazy-timestamp.test b/testdata/workloads/functional-query/queries/QueryTest/select-lazy-timestamp.test
index 8258072..d87e252 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/select-lazy-timestamp.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/select-lazy-timestamp.test
@@ -5,13 +5,6 @@ select ts from lazy_ts
2001-01-02 00:00:00
2001-01-02 00:00:00
2001-01-02 00:00:00
-01:06:08
-01:06:08
-01:06:08
-01:06:08
-01:06:08.123456789
-01:06:08.123456789
-01:06:08.123450000
2001-01-02 01:06:08
2001-01-02 01:06:08.123456000
2001-01-02 01:06:08.123456789
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index 52e921f..623298a 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -224,6 +224,6 @@ class TestTimestampErrors(TestDataErrors):
result = self.client.execute("SELECT * FROM " + FQ_TBL_NAME + " ORDER BY col")
assert len(result.data) == 7
assert result.data == ['1954-12-03 15:10:02', '1999-03-24 07:21:02', \
- '12:10:02', '15:03:09', 'NULL', 'NULL', 'NULL']
+ 'NULL', 'NULL', 'NULL', 'NULL', 'NULL']
result = self.client.execute("SELECT COUNT(DISTINCT col) FROM " + FQ_TBL_NAME)
- assert result.data == ['4']
+ assert result.data == ['2']
diff --git a/tests/query_test/test_cast_with_format.py b/tests/query_test/test_cast_with_format.py
index 9dbb102..504440c 100644
--- a/tests/query_test/test_cast_with_format.py
+++ b/tests/query_test/test_cast_with_format.py
@@ -2022,25 +2022,38 @@ class TestCastWithFormat(ImpalaTestSuite):
# Multiple fraction second token conflict
err = self.execute_query_expect_failure(self.client,
- "select cast('2018-10-10' as timestamp format 'FF FF1')")
+ "select cast('2018-10-10' as timestamp format 'YYYY-MM-DD FF FF1')")
assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
- "select cast('2018-10-10' as timestamp format 'FF2 FF3')")
+ "select cast('2018-10-10' as timestamp format 'YYYY-MM-DD FF2 FF3')")
assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
- "select cast('2018-10-10' as timestamp format 'FF4 FF5')")
+ "select cast('2018-10-10' as timestamp format 'YYYY-MM-DD FF4 FF5')")
assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
- "select cast('2018-10-10' as timestamp format 'FF6 FF7')")
+ "select cast('2018-10-10' as timestamp format 'YYYY-MM-DD FF6 FF7')")
assert "Multiple fractional second tokens provided." in str(err)
err = self.execute_query_expect_failure(self.client,
- "select cast('2018-10-10' as timestamp format 'FF8 FF9')")
+ "select cast('2018-10-10' as timestamp format 'YYYY-MM-DD FF8 FF9')")
assert "Multiple fractional second tokens provided." in str(err)
+ # No date token
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2020-05-05' as timestamp format 'FF1')")
+ assert "No date tokens provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2020-05-05' as timestamp format 'SSSSS')")
+ assert "No date tokens provided." in str(err)
+
+ err = self.execute_query_expect_failure(self.client,
+ "select cast('2020-05-05' as timestamp format 'HH:MI:SS')")
+ assert "No date tokens provided." in str(err)
+
# ISO 8601 Week-based and normal date pattern tokens must not be mixed.
err = self.execute_query_expect_failure(self.client,
"select cast('2018-10-01' as date format 'IYYY-MM-ID')")
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 02302d0..1ff5e6d 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -382,6 +382,19 @@ class TestParquet(ImpalaTestSuite):
self.run_test_case('QueryTest/out-of-range-timestamp-abort-on-error',
vector, unique_database)
+ def test_dateless_timestamp_parquet(self, vector, unique_database):
+ """Test scanning parquet files which still includes dateless timestamps."""
+ tbl_name = "timestamp_table"
+ create_sql = "create table %s.%s (t timestamp) stored as parquet" % (
+ unique_database, tbl_name)
+ create_table_and_copy_files(self.client, create_sql, unique_database, tbl_name,
+ ["/testdata/data/dateless_timestamps.parq"])
+
+ new_vector = deepcopy(vector)
+ del new_vector.get_value('exec_option')['abort_on_error']
+ self.run_test_case('QueryTest/dateless_timestamp_parquet', new_vector,
+ use_db=unique_database)
+
def test_date_out_of_range_parquet(self, vector, unique_database):
"""Test scanning parquet files with an out of range date."""
create_table_from_parquet(self.client, unique_database, "out_of_range_date")
@@ -1232,6 +1245,17 @@ class TestTextScanRangeLengths(ImpalaTestSuite):
self.run_test_case('QueryTest/hdfs-text-scan-with-header', new_vector,
test_file_vars={'$UNIQUE_DB': unique_database})
+ def test_dateless_timestamp_text(self, vector, unique_database):
+ """Test scanning text files which still includes dateless timestamps."""
+ tbl_name = "timestamp_text_table"
+ create_sql = "create table %s.%s (t timestamp) stored as textfile" % (
+ unique_database, tbl_name)
+ create_table_and_copy_files(self.client, create_sql, unique_database, tbl_name,
+ ["/testdata/data/dateless_timestamps.txt"])
+
+ new_vector = deepcopy(vector)
+ del new_vector.get_value('exec_option')['abort_on_error']
+ self.run_test_case('QueryTest/dateless_timestamp_text', new_vector, unique_database)
# Missing Coverage: No coverage for truncated files errors or scans.
@SkipIfS3.hive