You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2022/12/19 16:53:04 UTC
[hive] branch master updated: HIVE-26787: Pushdown Timestamp data type to metastore via directsql/JDO (Syed Shameerur Rahman, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ab4c53de82d HIVE-26787: Pushdown Timestamp data type to metastore via directsql/JDO (Syed Shameerur Rahman, reviewed by Krisztian Kasa)
ab4c53de82d is described below
commit ab4c53de82d4aaa33706510441167f2df55df15e
Author: Syed Shameerur Rahman <rh...@amazon.com>
AuthorDate: Mon Dec 19 22:22:49 2022 +0530
HIVE-26787: Pushdown Timestamp data type to metastore via directsql/JDO (Syed Shameerur Rahman, reviewed by Krisztian Kasa)
---
.../llap/temp_table_partition_timestamp.q.out | 130 ---------------------
.../hive/metastore/utils/MetaStoreUtils.java | 9 ++
.../hadoop/hive/metastore/DatabaseProduct.java | 8 ++
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 78 +++++++++----
.../hive/metastore/parser/ExpressionTree.java | 12 +-
.../apache/hadoop/hive/metastore/parser/Filter.g | 109 +++++++++++++++--
6 files changed, 180 insertions(+), 166 deletions(-)
diff --git a/ql/src/test/results/clientpositive/llap/temp_table_partition_timestamp.q.out b/ql/src/test/results/clientpositive/llap/temp_table_partition_timestamp.q.out
index 6ca919294a2..b67336e35b3 100644
--- a/ql/src/test/results/clientpositive/llap/temp_table_partition_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/temp_table_partition_timestamp.q.out
@@ -96,95 +96,45 @@ POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/r
PREHOOK: query: select * from partition_timestamp_1_temp where dt = '2000-01-01 01:00:00' and region = '2' order by key,value
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select * from partition_timestamp_1_temp where dt = '2000-01-01 01:00:00' and region = '2' order by key,value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 01:00:00'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 01:00:00'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
10
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = '2000-01-01 01:00:00'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = '2000-01-01 01:00:00'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
10
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 02:00:00' and region = '2'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 02:00:00' and region = '2'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
5
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2001-01-01 03:00:00' and region = '10'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2001-01-01 03:00:00' and region = '10'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
11
PREHOOK: query: select count(*) from partition_timestamp_1_temp where region = '1'
@@ -199,153 +149,73 @@ POSTHOOK: Input: default@partition_timestamp_1_temp
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 01:00:00' and region = '3'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '2000-01-01 01:00:00' and region = '3'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
0
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '1999-01-01 01:00:00'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt = timestamp '1999-01-01 01:00:00'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
0
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt > timestamp '2000-01-01 01:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt > timestamp '2000-01-01 01:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
20
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt < timestamp '2000-01-02 01:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt < timestamp '2000-01-02 01:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
10
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt >= timestamp '2000-01-02 01:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt >= timestamp '2000-01-02 01:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
20
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt <= timestamp '2000-01-01 01:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt <= timestamp '2000-01-01 01:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
10
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt <> timestamp '2000-01-01 01:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt <> timestamp '2000-01-01 01:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
20
PREHOOK: query: select count(*) from partition_timestamp_1_temp where dt between timestamp '1999-12-30 12:00:00' and timestamp '2000-01-03 12:00:00' and region = '1'
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_timestamp_1_temp
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-PREHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
POSTHOOK: query: select count(*) from partition_timestamp_1_temp where dt between timestamp '1999-12-30 12:00:00' and timestamp '2000-01-03 12:00:00' and region = '1'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_timestamp_1_temp
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 01%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2000-01-01 02%3A00%3A00/region=2
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 01%3A00%3A00/region=2020-20-20
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 02%3A00%3A00/region=1
-POSTHOOK: Input: default@partition_timestamp_1_temp@dt=2001-01-01 03%3A00%3A00/region=10
#### A masked pattern was here ####
10
PREHOOK: query: select count(*) from partition_timestamp_1_temp where region = '2020-20-20'
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index d41310906a7..0df401be7d1 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -25,6 +25,7 @@ import java.security.AccessController;
import java.security.PrivilegedAction;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -84,6 +85,14 @@ public class MetaStoreUtils {
return val;
}
};
+ public static final ThreadLocal<DateTimeFormatter> PARTITION_TIMESTAMP_FORMAT =
+ new ThreadLocal<DateTimeFormatter>() {
+ @Override
+ protected DateTimeFormatter initialValue() {
+ return DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").
+ withZone(TimeZone.getTimeZone("UTC").toZoneId());
+ }
+ };
// Indicates a type was derived from the deserializer rather than Hive's metadata.
public static final String TYPE_FROM_DESERIALIZER = "<derived from deserializer>";
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index 0c1511b5c58..301949c40f8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -247,6 +247,14 @@ public class DatabaseProduct implements Configurable {
}
}
+ protected String toTimestamp(String tableValue) {
+ if (isORACLE()) {
+ return "TO_TIMESTAMP(" + tableValue + ", 'YYYY-MM-DD HH:mm:ss')";
+ } else {
+ return "cast(" + tableValue + " as TIMESTAMP)";
+ }
+ }
+
/**
* Returns db-specific logic to be executed at the beginning of a transaction.
* Used in pooled connections.
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index e9ad23856de..d89e7ea87c7 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -21,11 +21,20 @@ package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang3.StringUtils.join;
import static org.apache.commons.lang3.StringUtils.normalizeSpace;
import static org.apache.commons.lang3.StringUtils.repeat;
+import static org.apache.hadoop.hive.metastore.ColumnType.BIGINT_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.DATE_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.INT_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.SMALLINT_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.STRING_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.TIMESTAMP_TYPE_NAME;
+import static org.apache.hadoop.hive.metastore.ColumnType.TINYINT_TYPE_NAME;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Timestamp;
import java.text.ParseException;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -45,6 +54,7 @@ import javax.jdo.Query;
import javax.jdo.Transaction;
import javax.jdo.datastore.JDOConnection;
+import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.StatsSetupConst;
@@ -553,6 +563,8 @@ class MetaStoreDirectSql {
PartitionFilterGenerator.FilterType.fromType(colType);
if (type == PartitionFilterGenerator.FilterType.Date) {
tableValue = dbType.toDate(tableValue);
+ } else if (type == PartitionFilterGenerator.FilterType.Timestamp) {
+ tableValue = dbType.toTimestamp(tableValue);
} else if (type == PartitionFilterGenerator.FilterType.Integral) {
tableValue = "CAST(" + tableColumn + " AS decimal(21,0))";
}
@@ -1248,30 +1260,43 @@ class MetaStoreDirectSql {
}
private static enum FilterType {
- Integral,
- String,
- Date,
-
- Invalid;
-
- static FilterType fromType(String colTypeStr) {
- if (colTypeStr.equals(ColumnType.STRING_TYPE_NAME)) {
- return FilterType.String;
- } else if (colTypeStr.equals(ColumnType.DATE_TYPE_NAME)) {
- return FilterType.Date;
- } else if (ColumnType.IntegralTypes.contains(colTypeStr)) {
- return FilterType.Integral;
+ Integral(ImmutableSet.of(TINYINT_TYPE_NAME, SMALLINT_TYPE_NAME, INT_TYPE_NAME, BIGINT_TYPE_NAME), Long.class),
+ String(ImmutableSet.of(STRING_TYPE_NAME), String.class),
+ Date(ImmutableSet.of(DATE_TYPE_NAME), java.sql.Date.class),
+ Timestamp(ImmutableSet.of(TIMESTAMP_TYPE_NAME), java.sql.Timestamp.class),
+
+ Invalid(Collections.emptySet(), Void.class);
+
+ private final Set<String> colTypes;
+ private final Class<?> clazz;
+
+ FilterType(Set<String> colTypes, Class<?> clazz) {
+ this.colTypes = colTypes;
+ this.clazz = clazz;
+ }
+
+ public Set<String> getType() {
+ return colTypes;
+ }
+
+ public Class<?> getClazz() {
+ return clazz;
+ }
+
+ public static FilterType fromType(String colTypeStr) {
+ for (FilterType filterType : FilterType.values()) {
+ if (filterType.colTypes.contains(colTypeStr)) {
+ return filterType;
+ }
}
return FilterType.Invalid;
}
- public static FilterType fromClass(Object value) {
- if (value instanceof String) {
- return FilterType.String;
- } else if (value instanceof Long) {
- return FilterType.Integral;
- } else if (value instanceof java.sql.Date) {
- return FilterType.Date;
+ public static FilterType fromClass(Object value){
+ for (FilterType filterType : FilterType.values()) {
+ if (filterType.clazz.isInstance(value)) {
+ return filterType;
+ }
}
return FilterType.Invalid;
}
@@ -1309,10 +1334,19 @@ class MetaStoreDirectSql {
}
}
+ if (colType == FilterType.Timestamp && valType == FilterType.String) {
+ nodeValue = Timestamp.valueOf(LocalDateTime.from(
+ MetaStoreUtils.PARTITION_TIMESTAMP_FORMAT.get().parse((String)nodeValue)));
+ valType = FilterType.Timestamp;
+ }
+
// We format it so we are sure we are getting the right value
if (valType == FilterType.Date) {
// Format
nodeValue = MetaStoreUtils.PARTITION_DATE_FORMAT.get().format(nodeValue);
+ } else if (valType == FilterType.Timestamp) {
+ //format
+ nodeValue = MetaStoreUtils.PARTITION_TIMESTAMP_FORMAT.get().format(((Timestamp) nodeValue).toLocalDateTime());
}
boolean isDefaultPartition = (valType == FilterType.String) && defaultPartName.equals(nodeValue);
@@ -1353,6 +1387,8 @@ class MetaStoreDirectSql {
tableValue = "cast(" + tableValue + " as decimal(21,0))";
} else if (colType == FilterType.Date) {
tableValue = dbType.toDate(tableValue);
+ } else if (colType == FilterType.Timestamp) {
+ tableValue = dbType.toTimestamp(tableValue);
}
// Workaround for HIVE_DEFAULT_PARTITION - ignore it like JDO does, for now.
@@ -1374,6 +1410,8 @@ class MetaStoreDirectSql {
if (valType == FilterType.Date) {
tableValue = dbType.toDate(tableValue);
+ } else if (valType == FilterType.Timestamp) {
+ tableValue = dbType.toTimestamp(tableValue);
}
}
if (!node.isReverseOrder) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
index 573d5a78cba..df1d7855665 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.metastore.parser;
+import java.sql.Timestamp;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -456,19 +457,22 @@ public class ExpressionTree {
boolean isIntegralSupported = canPushDownIntegral && canJdoUseStringsWithIntegral();
String colType = partitionKeys.get(partColIndex).getType();
// Can only support partitions whose types are string, or maybe integers
- // Date data type value is considered as string hence pushing down to JDO.
- if (!colType.equals(ColumnType.STRING_TYPE_NAME) && !colType.equals(ColumnType.DATE_TYPE_NAME)
+ // Date/Timestamp data type value is considered as string hence pushing down to JDO.
+ if (!ColumnType.STRING_TYPE_NAME.equalsIgnoreCase(colType) && !ColumnType.DATE_TYPE_NAME.equalsIgnoreCase(colType)
+ && !ColumnType.TIMESTAMP_TYPE_NAME.equalsIgnoreCase(colType)
&& (!isIntegralSupported || !ColumnType.IntegralTypes.contains(colType))) {
filterBuilder.setError("Filtering is supported only on partition keys of type " +
"string" + (isIntegralSupported ? ", or integral types" : ""));
return null;
}
- // There's no support for date cast in JDO. Let's convert it to string; the date
+ // There's no support for date or timestamp cast in JDO. Let's convert it to string; the date
// columns have been excluded above, so it will either compare w/string or fail.
Object val = value;
- if (value instanceof Date) {
+ if (colType.equals("date") && value instanceof Date) {
val = MetaStoreUtils.PARTITION_DATE_FORMAT.get().format((Date)value);
+ } else if (colType.equals("timestamp") && value instanceof Timestamp) {
+ val = MetaStoreUtils.PARTITION_TIMESTAMP_FORMAT.get().format(((Timestamp)value).toLocalDateTime());
}
boolean isStringValue = val instanceof String;
if (!isStringValue && (!isIntegralSupported || !(val instanceof Long))) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g
index 1573b0925c3..c3977860f71 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/Filter.g
@@ -36,6 +36,9 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
package org.apache.hadoop.hive.metastore.parser;
import java.sql.Date;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.TimeZone;
@@ -47,6 +50,8 @@ import java.util.regex.Pattern;
public String errorMsg;
private static final Pattern datePattern = Pattern.compile(".*(\\d\\d\\d\\d-\\d\\d-\\d\\d).*");
+ private static final Pattern timestampPattern =
+ Pattern.compile(".*(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d).*");
private static final ThreadLocal<SimpleDateFormat> dateFormat =
new ThreadLocal<SimpleDateFormat>() {
@Override
@@ -57,8 +62,17 @@ import java.util.regex.Pattern;
return val;
};
};
+ private static final ThreadLocal<DateTimeFormatter> timestampFormat =
+ new ThreadLocal<DateTimeFormatter>() {
+ @Override
+ protected DateTimeFormatter initialValue() {
+ DateTimeFormatter val = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+ .withZone(TimeZone.getTimeZone("UTC").toZoneId());
+ return val;
+ };
+ };
- public static java.sql.Date ExtractDate (String input) {
+ public static java.sql.Date extractDate(String input) {
Matcher m = datePattern.matcher(input);
if (!m.matches()) {
return null;
@@ -70,6 +84,19 @@ import java.util.regex.Pattern;
}
}
+ public static java.sql.Timestamp extractTimestamp(String input) {
+ Matcher m = timestampPattern.matcher(input);
+ if (!m.matches()) {
+ return null;
+ }
+ try {
+ LocalDateTime val = LocalDateTime.from(timestampFormat.get().parse(m.group(1)));
+ return Timestamp.valueOf(val);
+ } catch (Exception ex) {
+ return null;
+ }
+ }
+
@Override
public void emitErrorMessage(String msg) {
// save for caller to detect invalid filter
@@ -142,7 +169,13 @@ binOpExpression
(key = Identifier op = operator value = DateLiteral)
|
(value = DateLiteral op = operator key = Identifier) { isReverseOrder = true; }
- ) { val = FilterLexer.ExtractDate(value.getText()); }
+ ) { val = FilterLexer.extractDate(value.getText()); }
+ |
+ (
+ (key = Identifier op = operator value = TimestampLiteral)
+ |
+ (value = TimestampLiteral op = operator key = Identifier) { isReverseOrder = true; }
+ ) { val = FilterLexer.extractTimestamp(value.getText()); }
|
(
(key = Identifier op = operator value = StringLiteral)
@@ -184,8 +217,13 @@ betweenExpression
key = Identifier (KW_NOT { isPositive = false; } )? BETWEEN
(
(left = DateLiteral KW_AND right = DateLiteral) {
- leftV = FilterLexer.ExtractDate(left.getText());
- rightV = FilterLexer.ExtractDate(right.getText());
+ leftV = FilterLexer.extractDate(left.getText());
+ rightV = FilterLexer.extractDate(right.getText());
+ }
+ |
+ (left = TimestampLiteral KW_AND right = TimestampLiteral) {
+ leftV = FilterLexer.extractTimestamp(left.getText());
+ rightV = FilterLexer.extractTimestamp(right.getText());
}
|
(left = StringLiteral KW_AND right = StringLiteral) { leftV = TrimQuotes(left.getText());
@@ -222,18 +260,33 @@ inExpression
(
constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
constants.add(constantV);
}
(
COMMA constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
constants.add(constantV);
}
)*
)
|
+ (
+ constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
+ constants.add(constantV);
+ }
+ (
+ COMMA constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
+ constants.add(constantV);
+ }
+ )*
+ )
+ |
(
constant = StringLiteral
{
@@ -311,7 +364,12 @@ multiColInExpression
(
constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
+ partialConstants.add(constantV);
+ }
+ | constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
partialConstants.add(constantV);
}
| constant = StringLiteral
@@ -330,7 +388,12 @@ multiColInExpression
(
constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
+ partialConstants.add(constantV);
+ }
+ | constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
partialConstants.add(constantV);
}
| constant = StringLiteral
@@ -357,7 +420,12 @@ multiColInExpression
(
constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
+ partialConstants.add(constantV);
+ }
+ | constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
partialConstants.add(constantV);
}
| constant = StringLiteral
@@ -376,7 +444,12 @@ multiColInExpression
(
constant = DateLiteral
{
- constantV = FilterLexer.ExtractDate(constant.getText());
+ constantV = FilterLexer.extractDate(constant.getText());
+ partialConstants.add(constantV);
+ }
+ | constant = TimestampLiteral
+ {
+ constantV = FilterLexer.extractTimestamp(constant.getText());
partialConstants.add(constantV);
}
| constant = StringLiteral
@@ -425,6 +498,7 @@ KW_AND : 'AND';
KW_OR : 'OR';
KW_LIKE : 'LIKE';
KW_DATE : ('DATE'|'date');
+KW_TIMESTAMP : ('TIMESTAMP'|'timestamp');
KW_CONST : 'CONST';
KW_STRUCT : 'STRUCT';
@@ -458,13 +532,24 @@ fragment DateString
(Digit)(Digit)(Digit)(Digit) '-' (Digit)(Digit) '-' (Digit)(Digit)
;
+fragment TimestampString
+ :
+ (Digit)(Digit)(Digit)(Digit) '-' (Digit)(Digit) '-' (Digit)(Digit) ' ' (Digit)(Digit) ':' (Digit)(Digit) ':' (Digit)(Digit)
+ ;
+
/* When I figure out how to make lexer backtrack after validating predicate, dates would be able
to support single quotes [( '\'' DateString '\'' ) |]. For now, what we do instead is have a hack
to parse the string in metastore code from StringLiteral. */
DateLiteral
:
- KW_DATE '\'' DateString '\'' { ExtractDate(getText()) != null }?
- | DateString { ExtractDate(getText()) != null }?
+ KW_DATE '\'' DateString '\'' { extractDate(getText()) != null }?
+ | DateString { extractDate(getText()) != null }?
+ ;
+
+TimestampLiteral
+ :
+ KW_TIMESTAMP '\'' TimestampString '\'' { extractTimestamp(getText()) != null }?
+ | TimestampString { extractTimestamp(getText()) != null }?
;
StringLiteral