You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2016/01/19 01:03:51 UTC
hive git commit: Vectorization:
VectorCopyRow/VectorAssignRow/VectorDeserializeRow assign needs explicit
isNull[offset] modification (Gopal V, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 36e855084 -> 9cab4414c
Vectorization: VectorCopyRow/VectorAssignRow/VectorDeserializeRow assign needs explicit isNull[offset] modification (Gopal V, reviewed by Sergey Shelukhin)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9cab4414
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9cab4414
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9cab4414
Branch: refs/heads/master
Commit: 9cab4414caf1bba2eb1852536a9d3676ba7eab21
Parents: 36e8550
Author: Gopal V <go...@apache.org>
Authored: Mon Jan 18 16:03:40 2016 -0800
Committer: Gopal V <go...@apache.org>
Committed: Mon Jan 18 16:03:40 2016 -0800
----------------------------------------------------------------------
.../hive/ql/exec/vector/VectorAssignRow.java | 16 ++++++++++++++++
.../hive/ql/exec/vector/VectorCopyRow.java | 10 ++++++++++
.../ql/exec/vector/VectorDeserializeRow.java | 20 ++++++++++++++++++++
.../ql/exec/vector/TestVectorRowObject.java | 7 +++++++
.../hive/ql/exec/vector/TestVectorSerDeRow.java | 11 +++++++++++
5 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9cab4414/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index 92b4a07..d69454f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -139,6 +139,7 @@ public abstract class VectorAssignRow {
} else {
BooleanWritable bw = (BooleanWritable) object;
vector[batchIndex] = (bw.get() ? 1 : 0);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -156,6 +157,7 @@ public abstract class VectorAssignRow {
} else {
ByteWritable bw = (ByteWritable) object;
vector[batchIndex] = bw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -173,6 +175,7 @@ public abstract class VectorAssignRow {
} else {
ShortWritable sw = (ShortWritable) object;
vector[batchIndex] = sw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -190,6 +193,7 @@ public abstract class VectorAssignRow {
} else {
IntWritable iw = (IntWritable) object;
vector[batchIndex] = iw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -207,6 +211,7 @@ public abstract class VectorAssignRow {
} else {
LongWritable lw = (LongWritable) object;
vector[batchIndex] = lw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -224,6 +229,7 @@ public abstract class VectorAssignRow {
} else {
DateWritable bw = (DateWritable) object;
vector[batchIndex] = bw.getDays();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -242,6 +248,7 @@ public abstract class VectorAssignRow {
TimestampWritable tw = (TimestampWritable) object;
Timestamp t = tw.getTimestamp();
vector[batchIndex] = TimestampUtils.getTimeNanoSec(t);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -260,6 +267,7 @@ public abstract class VectorAssignRow {
HiveIntervalYearMonthWritable iymw = (HiveIntervalYearMonthWritable) object;
HiveIntervalYearMonth iym = iymw.getHiveIntervalYearMonth();
vector[batchIndex] = iym.getTotalMonths();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -278,6 +286,7 @@ public abstract class VectorAssignRow {
HiveIntervalDayTimeWritable idtw = (HiveIntervalDayTimeWritable) object;
HiveIntervalDayTime idt = idtw.getHiveIntervalDayTime();
vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(idt);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -317,6 +326,7 @@ public abstract class VectorAssignRow {
} else {
FloatWritable fw = (FloatWritable) object;
vector[batchIndex] = fw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -334,6 +344,7 @@ public abstract class VectorAssignRow {
} else {
DoubleWritable dw = (DoubleWritable) object;
vector[batchIndex] = dw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -370,6 +381,7 @@ public abstract class VectorAssignRow {
} else {
BytesWritable bw = (BytesWritable) object;
colVector.setVal(batchIndex, bw.getBytes(), 0, bw.getLength());
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -387,6 +399,7 @@ public abstract class VectorAssignRow {
} else {
Text tw = (Text) object;
colVector.setVal(batchIndex, tw.getBytes(), 0, tw.getLength());
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -411,6 +424,7 @@ public abstract class VectorAssignRow {
}
byte[] bytes = hiveVarchar.getValue().getBytes();
colVector.setVal(batchIndex, bytes, 0, bytes.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -437,6 +451,7 @@ public abstract class VectorAssignRow {
// We store CHAR in vector row batch with padding stripped.
byte[] bytes = hiveChar.getStrippedValue().getBytes();
colVector.setVal(batchIndex, bytes, 0, bytes.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -469,6 +484,7 @@ public abstract class VectorAssignRow {
} else {
colVector.set(batchIndex, (HiveDecimalWritable) object);
}
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9cab4414/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index c56903e..6b681b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -60,12 +60,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.vector[outBatchIndex] = inColVector.vector[0];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -87,12 +89,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.vector[outBatchIndex] = inColVector.vector[0];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -122,12 +126,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.setVal(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.setVal(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -149,12 +155,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.setRef(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.setRef(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -175,12 +183,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.set(outBatchIndex, inColVector.vector[0]);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.set(outBatchIndex, inColVector.vector[inBatchIndex]);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
http://git-wip-us.apache.org/repos/asf/hive/blob/9cab4414/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index 4d86db6..9b086b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -102,6 +102,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
boolean value = deserializeRead.readBoolean();
colVector.vector[batchIndex] = (value ? 1 : 0);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -121,6 +122,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
byte value = deserializeRead.readByte();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -140,6 +142,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
short value = deserializeRead.readShort();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -159,6 +162,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
int value = deserializeRead.readInt();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -178,6 +182,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
long value = deserializeRead.readLong();
colVector.vector[batchIndex] = value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -200,6 +205,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
deserializeRead.readDate(readDateResults);
colVector.vector[batchIndex] = (long) readDateResults.getDays();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -223,6 +229,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readTimestamp(readTimestampResults);
Timestamp t = readTimestampResults.getTimestamp();
colVector.vector[batchIndex] = TimestampUtils.getTimeNanoSec(t);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -246,6 +253,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readIntervalYearMonth(readIntervalYearMonthResults);
HiveIntervalYearMonth hiym = readIntervalYearMonthResults.getHiveIntervalYearMonth();
colVector.vector[batchIndex] = hiym.getTotalMonths();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -269,6 +277,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readIntervalDayTime(readIntervalDayTimeResults);
HiveIntervalDayTime hidt = readIntervalDayTimeResults.getHiveIntervalDayTime();
colVector.vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(hidt);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -295,6 +304,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
float value = deserializeRead.readFloat();
colVector.vector[batchIndex] = (double) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -314,6 +324,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
} else {
double value = deserializeRead.readDouble();
colVector.vector[batchIndex] = value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -344,6 +355,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readString(readStringResults);
colVector.setVal(batchIndex, readStringResults.bytes,
readStringResults.start, readStringResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -367,6 +379,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readString(readStringResults);
colVector.setRef(batchIndex, readStringResults.bytes,
readStringResults.start, readStringResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -396,6 +409,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, charTypeInfo.getLength());
colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -425,6 +439,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, charTypeInfo.getLength());
colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -454,6 +469,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
int adjustedLength = StringExpr.truncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, varcharTypeInfo.getLength());
colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -483,6 +499,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
int adjustedLength = StringExpr.truncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, varcharTypeInfo.getLength());
colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -506,6 +523,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readBinary(readBinaryResults);
colVector.setVal(batchIndex, readBinaryResults.bytes,
readBinaryResults.start, readBinaryResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -529,6 +547,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readBinary(readBinaryResults);
colVector.setRef(batchIndex, readBinaryResults.bytes,
readBinaryResults.start, readBinaryResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -552,6 +571,7 @@ public final class VectorDeserializeRow<T extends DeserializeRead> {
deserializeRead.readHiveDecimal(readDecimalResults);
HiveDecimal hiveDecimal = readDecimalResults.getHiveDecimal();
colVector.vector[batchIndex].set(hiveDecimal);
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9cab4414/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
index c076e6c..a5946d1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -60,6 +61,12 @@ public class TestVectorRowObject extends TestCase {
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+ // junk the destination for the 1st pass
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ cv.noNulls = false;
+ }
+
VectorAssignRowSameBatch vectorAssignRow = new VectorAssignRowSameBatch();
vectorAssignRow.init(source.typeNames());
vectorAssignRow.setOneBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/9cab4414/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
index ae4b239..7c0c8d1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
@@ -563,6 +563,11 @@ public class TestVectorSerDeRow extends TestCase {
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+ // junk the destination for the 1st pass
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ }
+
int fieldCount = source.typeNames().size();
DeserializeRead deserializeRead;
SerializeWrite serializeWrite;
@@ -592,6 +597,12 @@ public class TestVectorSerDeRow extends TestCase {
VectorDeserializeRow vectorDeserializeRow = new VectorDeserializeRow(deserializeRead);
vectorDeserializeRow.init();
+ // junk the destination for the 1st pass
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ cv.noNulls = false;
+ }
+
VectorExtractRowSameBatch vectorExtractRow = new VectorExtractRowSameBatch();
vectorExtractRow.init(source.typeNames());
vectorExtractRow.setOneBatch(batch);