You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2015/04/23 20:26:40 UTC
[6/6] hive git commit: HIVE-9917: After HIVE-3454 is done,
make int to timestamp conversion configurable (Aihua Xu via Jason
Dere)
HIVE-9917: After HIVE-3454 is done, make int to timestamp conversion configurable (Aihua Xu via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84494ad6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84494ad6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84494ad6
Branch: refs/heads/master
Commit: 84494ad63150557147e09efbba2056e0f78b438d
Parents: fd65c4e
Author: Jason Dere <jd...@hortonworks.com>
Authored: Thu Apr 23 11:26:04 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Thu Apr 23 11:26:04 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ant/GenVectorCode.java | 2 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../src/test/queries/positive/hbase_timestamp.q | 22 +-
.../test/results/positive/hbase_timestamp.q.out | 60 +-
.../ql/exec/vector/VectorizationContext.java | 16 +-
.../ql/exec/vector/expressions/MathExpr.java | 9 +-
.../ql/udf/generic/GenericUDFTimestamp.java | 24 +
.../clientpositive/timestamp_ints_casts.q | 74 +
.../queries/clientpositive/vectorized_casts.q | 2 +-
.../vectorized_timestamp_ints_casts.q | 76 +
.../spark/vector_between_in.q.out | 6088 +-----------------
.../spark/vectorization_decimal_date.q.out | 20 +-
.../clientpositive/tez/vector_between_in.q.out | 6088 +-----------------
.../tez/vectorization_decimal_date.q.out | 20 +-
.../clientpositive/tez/vectorized_casts.q.out | 52 +-
.../clientpositive/timestamp_ints_casts.q.out | 264 +
.../clientpositive/vector_between_in.q.out | 6088 +-----------------
.../vectorization_decimal_date.q.out | 20 +-
.../clientpositive/vectorized_casts.q.out | 52 +-
.../vectorized_timestamp_ints_casts.q.out | 266 +
.../hive/serde2/io/TimestampWritable.java | 11 +-
.../PrimitiveObjectInspectorConverter.java | 7 +-
.../PrimitiveObjectInspectorUtils.java | 42 +-
.../TestPrimitiveObjectInspectorUtils.java | 95 +-
24 files changed, 1277 insertions(+), 18124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
index f7575a3..6485a2a 100644
--- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
+++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
@@ -771,6 +771,8 @@ public class GenVectorCode extends Task {
"", "", "timestamp"},
{"ColumnUnaryFunc", "CastLongToTimestampVia", "long", "long", "MathExpr.longToTimestamp", "",
"", "", ""},
+ {"ColumnUnaryFunc", "CastMillisecondsLongToTimestampVia", "long", "long", "MathExpr.millisecondsLongToTimestamp", "",
+ "", "", ""},
{"ColumnUnaryFunc", "CastDoubleToTimestampVia", "long", "double",
"MathExpr.doubleToTimestamp", "", "", "", ""},
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a3bd9aa..c9ee423 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -919,6 +919,9 @@ public class HiveConf extends Configuration {
HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION("hive.parquet.timestamp.skip.conversion", true,
"Current Hive implementation of parquet stores timestamps to UTC, this flag allows skipping of the conversion" +
"on reading parquet files from other tools"),
+ HIVE_INT_TIMESTAMP_CONVERSION_IN_SECONDS("hive.int.timestamp.conversion.in.seconds", false,
+ "Boolean/tinyint/smallint/int/bigint value is interpreted as milliseconds during the timestamp conversion.\n" +
+ "Set this flag to true to interpret the value as seconds to be consistent with float/double." ),
HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f,
"Maximum fraction of heap that can be used by ORC file writers"),
HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null,
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/hbase-handler/src/test/queries/positive/hbase_timestamp.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_timestamp.q b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
index 1a2cbdf..6f1e205 100644
--- a/hbase-handler/src/test/queries/positive/hbase_timestamp.q
+++ b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
@@ -10,7 +10,7 @@ DROP TABLE hbase_table;
CREATE TABLE hbase_table (key string, value string, time bigint)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp");
-FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754 WHERE (key % 17) = 0;
+FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0;
SELECT key, value, cast(time as timestamp) FROM hbase_table;
DROP TABLE hbase_table;
@@ -19,23 +19,23 @@ CREATE TABLE hbase_table (key string, value string, time bigint)
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp");
insert overwrite table hbase_table select key,value,ts FROM
(
- select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+ select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
UNION ALL
- select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+ select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
) T;
explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000;
explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000;
explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000;
explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000;
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
index 45c15ec..3aae7d0 100644
--- a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
@@ -81,11 +81,11 @@ POSTHOOK: query: CREATE TABLE hbase_table (key string, value string, time bigint
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hbase_table
-PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754 WHERE (key % 17) = 0
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@hbase_table
-POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754 WHERE (key % 17) = 0
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key % 17) = 0
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@hbase_table
@@ -139,27 +139,27 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@hbase_table
PREHOOK: query: insert overwrite table hbase_table select key,value,ts FROM
(
- select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+ select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
UNION ALL
- select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+ select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
) T
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@hbase_table
POSTHOOK: query: insert overwrite table hbase_table select key,value,ts FROM
(
- select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+ select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
UNION ALL
- select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+ select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
) T
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@hbase_table
PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000
PREHOOK: type: QUERY
POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -171,10 +171,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hbase_table
- filterExpr: (((key > 100.0) and (key < 400.0)) and (time < 200000000)) (type: boolean)
+ filterExpr: (((key > 100.0) and (key < 400.0)) and (time < 200000000000)) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time < 200000000))) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time < 200000000000))) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp)
@@ -194,21 +194,21 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time < 200000000000
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table
#### A masked pattern was here ####
165 val_165 1973-03-03 01:46:40
396 val_396 1973-03-03 01:46:40
PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000
PREHOOK: type: QUERY
POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -220,10 +220,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hbase_table
- filterExpr: (((key > 100.0) and (key < 400.0)) and (time > 100000000)) (type: boolean)
+ filterExpr: (((key > 100.0) and (key < 400.0)) and (time > 100000000000)) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time > 100000000))) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time > 100000000000))) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp)
@@ -243,11 +243,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time > 100000000000
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table
#### A masked pattern was here ####
@@ -256,10 +256,10 @@ POSTHOOK: Input: default@hbase_table
296 val_296 1976-05-03 12:33:20
333 val_333 1976-05-03 12:33:20
PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000
PREHOOK: type: QUERY
POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -271,10 +271,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hbase_table
- filterExpr: (((key > 100.0) and (key < 400.0)) and (time <= 100000000)) (type: boolean)
+ filterExpr: (((key > 100.0) and (key < 400.0)) and (time <= 100000000000)) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time <= 100000000))) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time <= 100000000000))) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp)
@@ -294,21 +294,21 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time <= 100000000000
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table
#### A masked pattern was here ####
165 val_165 1973-03-03 01:46:40
396 val_396 1973-03-03 01:46:40
PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000
PREHOOK: type: QUERY
POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -320,10 +320,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: hbase_table
- filterExpr: (((key > 100.0) and (key < 400.0)) and (time >= 200000000)) (type: boolean)
+ filterExpr: (((key > 100.0) and (key < 400.0)) and (time >= 200000000000)) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time >= 200000000))) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0) and (time >= 200000000000))) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP) (type: timestamp)
@@ -343,11 +343,11 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table
#### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key < 400 AND time >= 200000000000
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index abce606..df39218 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1152,8 +1152,9 @@ public class VectorizationContext {
return getCastToChar(childExpr, returnType);
} else if (udf instanceof GenericUDFToVarchar) {
return getCastToVarChar(childExpr, returnType);
+ } else if (udf instanceof GenericUDFTimestamp) {
+ return getCastToTimestamp((GenericUDFTimestamp)udf, childExpr, mode, returnType);
}
-
// Now do a general lookup
Class<?> udfClass = udf.getClass();
if (udf instanceof GenericUDFBridge) {
@@ -1169,6 +1170,19 @@ public class VectorizationContext {
return ve;
}
+ private VectorExpression getCastToTimestamp(GenericUDFTimestamp udf,
+ List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
+ VectorExpression ve = getVectorExpressionForUdf(udf.getClass(), childExpr, mode, returnType);
+
+ // Replace with the milliseconds conversion
+ if (!udf.isIntToTimestampInSeconds() && ve instanceof CastLongToTimestampViaLongToLong) {
+ ve = createVectorExpression(CastMillisecondsLongToTimestampViaLongToLong.class,
+ childExpr, Mode.PROJECTION, returnType);
+ }
+
+ return ve;
+ }
+
private VectorExpression getCoalesceExpression(List<ExprNodeDesc> childExpr, TypeInfo returnType)
throws HiveException {
int[] inputColumns = new int[childExpr.size()];
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
index 3709aa3..aef923e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.ql.exec.vector.expressions;
-import java.io.IOException;
-import java.io.OutputStream;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
/**
@@ -82,6 +80,13 @@ public class MathExpr {
return v * 1000000000;
}
+ /* Convert an integer value in milliseconds since the epoch to a timestamp value
+ * for use in a long column vector, which is represented in nanoseconds since the epoch.
+ */
+ public static long millisecondsLongToTimestamp(long v) {
+ return v * 1000000;
+ }
+
// Convert seconds since the epoch (with fraction) to nanoseconds, as a long integer.
public static long doubleToTimestamp(double v) {
return (long)( v * 1000000000.0);
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
index 2f854f4..24b49a0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
@@ -25,6 +28,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToTimestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToTimestampViaDoubleToLong;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToTimestampViaLongToLong;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter;
@@ -48,6 +53,14 @@ public class GenericUDFTimestamp extends GenericUDF {
private transient PrimitiveObjectInspector argumentOI;
private transient TimestampConverter tc;
+ /*
+ * Integer value was interpreted to timestamp inconsistently in milliseconds comparing
+ * to float/double in seconds. Since the issue exists for a long time and some users may
+ * use in such inconsistent way, use the following flag to keep backward compatible.
+ * If the flag is set to false, integer value is interpreted as timestamp in milliseconds;
+ * otherwise, it's interpreted as timestamp in seconds.
+ */
+ private boolean intToTimestampInSeconds = false;
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
@@ -56,6 +69,12 @@ public class GenericUDFTimestamp extends GenericUDF {
"The function TIMESTAMP requires at least one argument, got "
+ arguments.length);
}
+
+ SessionState ss = SessionState.get();
+ if (ss != null) {
+ intToTimestampInSeconds = ss.getConf().getBoolVar(ConfVars.HIVE_INT_TIMESTAMP_CONVERSION_IN_SECONDS);
+ }
+
try {
argumentOI = (PrimitiveObjectInspector) arguments[0];
} catch (ClassCastException e) {
@@ -65,6 +84,8 @@ public class GenericUDFTimestamp extends GenericUDF {
tc = new TimestampConverter(argumentOI,
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
+ tc.setIntToTimestampInSeconds(intToTimestampInSeconds);
+
return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
}
@@ -88,4 +109,7 @@ public class GenericUDFTimestamp extends GenericUDF {
return sb.toString();
}
+ public boolean isIntToTimestampInSeconds() {
+ return intToTimestampInSeconds;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/timestamp_ints_casts.q b/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
new file mode 100644
index 0000000..b043d59
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
@@ -0,0 +1,74 @@
+set hive.int.timestamp.conversion.in.seconds=false;
+
+explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+set hive.int.timestamp.conversion.in.seconds=true;
+
+explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/vectorized_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_casts.q b/ql/src/test/queries/clientpositive/vectorized_casts.q
index 3f818b1..9147a6a 100644
--- a/ql/src/test/queries/clientpositive/vectorized_casts.q
+++ b/ql/src/test/queries/clientpositive/vectorized_casts.q
@@ -146,4 +146,4 @@ from alltypesorc
-- limit output to a reasonably small number of rows
where cbigint % 250 = 0;
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q b/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
new file mode 100644
index 0000000..f297070
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
@@ -0,0 +1,76 @@
+SET hive.vectorized.execution.enabled = true;
+SET hive.int.timestamp.conversion.in.seconds=false;
+
+explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+SET hive.int.timestamp.conversion.in.seconds=true;
+
+explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;