You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2019/07/25 19:12:31 UTC
[hive] branch master updated: HIVE-21862: ORC ppd produces wrong
result with timestamp (Vineet Garg, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
vgarg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new bc35a37 HIVE-21862: ORC ppd produces wrong result with timestamp (Vineet Garg, reviewed by Jesus Camacho Rodriguez)
bc35a37 is described below
commit bc35a371610092ba423b739ca171a2e56003da27
Author: Vineet Garg <vg...@apache.org>
AuthorDate: Thu Jul 25 12:11:39 2019 -0700
HIVE-21862: ORC ppd produces wrong result with timestamp (Vineet Garg, reviewed by Jesus Camacho Rodriguez)
---
.../hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java | 2 +-
.../apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java | 3 +++
.../hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java | 3 ++-
.../change_allowincompatible_vectorization_false_date.q | 2 ++
.../change_allowincompatible_vectorization_false_date.q.out | 11 +++++++++++
6 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 6a00220..85a42f9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -782,7 +782,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
sargApp = new RecordReaderImpl.SargApplier(sarg,
rowIndexStride, evolution,
OrcFile.WriterVersion.from(OrcFile.WriterImplementation.ORC_JAVA, fileMetadata.getWriterVersionNum()),
- false);
+ true);
}
boolean hasAnyData = false;
// stripeRgs should have been initialized by this time with an empty array.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index ec4994d..f72705f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -2313,7 +2313,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
PredicateLeaf leaf = predLeaves.get(pred);
try {
- truthValues[pred] = RecordReaderImpl.evaluatePredicate(stats, leaf, null);
+ truthValues[pred] = RecordReaderImpl.evaluatePredicate(stats, leaf, null, true);
} catch (NoDynamicValuesException dve) {
LOG.debug("Dynamic values are not available here {}", dve.getMessage());
boolean hasNulls = stats.hasNull() || leaf.getOperator() != Operator.NULL_SAFE_EQUALS;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
index 27fe828..bfabdce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
@@ -204,6 +204,9 @@ public class ConvertAstToSearchArg {
}
return fl.doubleValue();
case TIMESTAMP:
+ if (lit instanceof org.apache.hadoop.hive.common.type.Timestamp) {
+ return ((org.apache.hadoop.hive.common.type.Timestamp) lit).toSqlTimestamp();
+ }
return Timestamp.valueOf(lit.toString());
case DATE:
return Date.valueOf(lit.toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
index 35053e3..3a40b31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java
@@ -251,7 +251,8 @@ public class FixedBucketPruningOptimizer extends Transform {
if (o instanceof java.sql.Date) {
return Date.valueOf(o.toString());
} else if (o instanceof java.sql.Timestamp) {
- return Timestamp.valueOf(o.toString());
+ java.sql.Timestamp sqlTimestamp = (java.sql.Timestamp)o;
+ return Timestamp.ofEpochMilli(sqlTimestamp.getTime(), sqlTimestamp.getNanos());
}
return o;
}
diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q
index e750c59..02d923b 100644
--- a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q
+++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q
@@ -27,3 +27,5 @@ insert into table change_allowincompatible_vectorization_false_date partition (s
select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa';
+set hive.vectorized.execution.enabled=true;
+select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa';
diff --git a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
index 4661a03..f3716b1 100644
--- a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
+++ b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out
@@ -84,3 +84,14 @@ POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date
POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
#### A masked pattern was here ####
2038-03-22 07:26:48
+PREHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@change_allowincompatible_vectorization_false_date
+PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
+#### A masked pattern was here ####
+POSTHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date
+POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa
+#### A masked pattern was here ####
+2038-03-22 07:26:48