You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/10/27 19:03:12 UTC
svn commit: r1634629 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/orc/ test/queries/clientpositive/
test/results/clientpositive/ test/results/clientpositive/tez/
Author: gunther
Date: Mon Oct 27 18:03:11 2014
New Revision: 1634629
URL: http://svn.apache.org/r1634629
Log:
HIVE-8566: Vectorized queries output wrong timestamps (Matt McCline reviewed by Prasanth J)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/trunk/ql/src/test/queries/clientpositive/vector_data_types.q
hive/trunk/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_data_types.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1634629&r1=1634628&r2=1634629&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Mon Oct 27 18:03:11 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
@@ -1074,41 +1075,20 @@ class RecordReaderImpl implements Record
result = (LongColumnVector) previousVector;
}
- // Read present/isNull stream
- super.nextVector(result, batchSize);
-
- data.nextVector(result, batchSize);
- nanoVector.isNull = result.isNull;
- nanos.nextVector(nanoVector, batchSize);
-
- if(result.isRepeating && nanoVector.isRepeating) {
- batchSize = 1;
- }
-
- // Non repeating values preset in the vector. Iterate thru the vector and populate the time
+ result.reset();
+ Object obj = null;
for (int i = 0; i < batchSize; i++) {
- if (!result.isNull[i]) {
- long ms = (result.vector[result.isRepeating ? 0 : i] + WriterImpl.BASE_TIMESTAMP)
- * WriterImpl.MILLIS_PER_SECOND;
- long ns = parseNanos(nanoVector.vector[nanoVector.isRepeating ? 0 : i]);
- // the rounding error exists because java always rounds up when dividing integers
- // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000)
- // to get the correct value we need
- // (-42 - 1)*1000 + 999 = -42001
- // (42)*1000 + 1 = 42001
- if(ms < 0 && ns != 0) {
- ms -= 1000;
- }
- // Convert millis into nanos and add the nano vector value to it
- result.vector[i] = (ms * 1000000) + ns;
+ obj = next(obj);
+ if (obj == null) {
+ result.noNulls = false;
+ result.isNull[i] = true;
+ } else {
+ TimestampWritable writable = (TimestampWritable) obj;
+ Timestamp timestamp = writable.getTimestamp();
+ result.vector[i] = TimestampUtils.getTimeNanoSec(timestamp);
}
}
- if(!(result.isRepeating && nanoVector.isRepeating)) {
- // both have to repeat for the result to be repeating
- result.isRepeating = false;
- }
-
return result;
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/vector_data_types.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_data_types.q?rev=1634629&r1=1634628&r2=1634629&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_data_types.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_data_types.q Mon Oct 27 18:03:11 2014
@@ -40,8 +40,14 @@ EXPLAIN SELECT t, si, i, b, f, d, bo, s,
SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20;
+SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q;
+
SET hive.vectorized.execution.enabled=true;
EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20;
-SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20;
\ No newline at end of file
+SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20;
+
+SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q;
\ No newline at end of file
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vector_data_types.q.out?rev=1634629&r1=1634628&r2=1634629&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/vector_data_types.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/vector_data_types.q.out Mon Oct 27 18:03:11 2014
@@ -177,6 +177,17 @@ NULL 409 65536 4294967490 46.97 25.92 fa
120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty
71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology
114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20
@@ -239,23 +250,34 @@ POSTHOOK: query: SELECT t, si, i, b, f,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
-108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided
-118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication
-18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topology
-69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topology
-27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biology
-42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american history
-NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history
-59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studies
-63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfing
-121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard duty
-72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topology
-22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecided
-101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biology
-109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skills
-83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hour
-29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studies
-70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggying
-120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard duty
-71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geology
-114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone band
+108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided
+118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication
+18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology
+69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology
+27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology
+42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history
+NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history
+59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies
+63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing
+121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty
+72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology
+22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided
+101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology
+109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills
+83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour
+29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies
+70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying
+120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty
+71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology
+114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
Modified: hive/trunk/ql/src/test/results/clientpositive/vector_data_types.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_data_types.q.out?rev=1634629&r1=1634628&r2=1634629&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_data_types.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_data_types.q.out Mon Oct 27 18:03:11 2014
@@ -171,6 +171,17 @@ NULL 409 65536 4294967490 46.97 25.92 fa
120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty
71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology
114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT 20
@@ -226,23 +237,34 @@ POSTHOOK: query: SELECT t, si, i, b, f,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
-108 301 65536 4294967357 90.05 17.59 true ethan johnson 1860-11-12 20:05:55.011470936 75.70 undecided
-118 497 65536 4294967381 50.32 12.72 false david nixon 1860-11-12 20:05:55.011484936 83.48 values clariffication
-18 280 65536 4294967320 32.92 45.94 false holly white 1860-11-12 20:05:55.011285936 58.86 topology
-69 489 65536 4294967404 33.52 17.99 false oscar ichabod 1860-11-12 20:05:55.011446936 32.68 topology
-27 405 65536 4294967508 82.24 29.41 true oscar ovid 1860-11-12 20:05:55.011365936 16.85 biology
-42 495 65536 4294967431 43.57 46.81 false tom johnson 1860-11-12 20:05:55.011444936 62.25 american history
-NULL 409 65536 4294967490 46.97 25.92 false fred miller 1860-11-12 20:05:55.011315936 33.45 history
-59 431 65537 4294967326 11.34 2.9 true oscar robinson 1860-11-12 20:05:55.011360936 81.04 zync studies
-63 458 65537 4294967463 21.94 49.71 true fred van buren 1860-11-12 20:05:55.011477936 99.34 wind surfing
-121 355 65537 4294967437 85.9 10.99 true calvin brown 1860-11-12 20:05:55.011453936 4.49 yard duty
-72 267 65537 4294967460 55.2 42.89 false oscar carson 1860-11-12 20:05:55.011352936 51.91 topology
-22 264 65537 4294967419 8.07 10.71 false david xylophone 1860-11-12 20:05:55.011335936 11.87 undecided
-101 283 65538 4294967527 84.03 27.29 true yuri brown 1860-11-12 20:05:55.011505936 44.24 biology
-109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 1860-11-12 20:05:55.011485936 9.53 study skills
-83 353 65538 4294967339 0.08 23.91 true holly thompson 1860-11-12 20:05:55.011356936 30.27 quiet hour
-29 331 65539 4294967420 73.18 28.96 true ethan brown 1860-11-12 20:05:55.011293936 58.85 zync studies
-70 430 65539 4294967534 18.89 43.84 true tom carson 1860-11-12 20:05:55.011381936 21.93 joggying
-120 331 65539 4294967324 88.02 40.94 true holly nixon 1860-11-12 20:05:55.011461936 96.64 yard duty
-71 298 65540 4294967510 91.63 9.6 false gabriella zipper 1860-11-12 20:05:55.011339936 68.14 geology
-114 385 65541 4294967458 73.48 34.97 true oscar quirinius 1860-11-12 20:05:55.011342936 72.33 xylophone band
+108 301 65536 4294967357 90.05 17.59 true ethan johnson 2013-03-01 09:11:58.703271 75.70 undecided
+118 497 65536 4294967381 50.32 12.72 false david nixon 2013-03-01 09:11:58.703285 83.48 values clariffication
+18 280 65536 4294967320 32.92 45.94 false holly white 2013-03-01 09:11:58.703086 58.86 topology
+69 489 65536 4294967404 33.52 17.99 false oscar ichabod 2013-03-01 09:11:58.703247 32.68 topology
+27 405 65536 4294967508 82.24 29.41 true oscar ovid 2013-03-01 09:11:58.703166 16.85 biology
+42 495 65536 4294967431 43.57 46.81 false tom johnson 2013-03-01 09:11:58.703245 62.25 american history
+NULL 409 65536 4294967490 46.97 25.92 false fred miller 2013-03-01 09:11:58.703116 33.45 history
+59 431 65537 4294967326 11.34 2.9 true oscar robinson 2013-03-01 09:11:58.703161 81.04 zync studies
+63 458 65537 4294967463 21.94 49.71 true fred van buren 2013-03-01 09:11:58.703278 99.34 wind surfing
+121 355 65537 4294967437 85.9 10.99 true calvin brown 2013-03-01 09:11:58.703254 4.49 yard duty
+72 267 65537 4294967460 55.2 42.89 false oscar carson 2013-03-01 09:11:58.703153 51.91 topology
+22 264 65537 4294967419 8.07 10.71 false david xylophone 2013-03-01 09:11:58.703136 11.87 undecided
+101 283 65538 4294967527 84.03 27.29 true yuri brown 2013-03-01 09:11:58.703306 44.24 biology
+109 376 65538 4294967453 59.61 35.62 true priscilla xylophone 2013-03-01 09:11:58.703286 9.53 study skills
+83 353 65538 4294967339 0.08 23.91 true holly thompson 2013-03-01 09:11:58.703157 30.27 quiet hour
+29 331 65539 4294967420 73.18 28.96 true ethan brown 2013-03-01 09:11:58.703094 58.85 zync studies
+70 430 65539 4294967534 18.89 43.84 true tom carson 2013-03-01 09:11:58.703182 21.93 joggying
+120 331 65539 4294967324 88.02 40.94 true holly nixon 2013-03-01 09:11:58.703262 96.64 yard duty
+71 298 65540 4294967510 91.63 9.6 false gabriella zipper 2013-03-01 09:11:58.70314 68.14 geology
+114 385 65541 4294967458 73.48 34.97 true oscar quirinius 2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i) as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556