You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/04/30 01:37:07 UTC
hive git commit: Vectorization:
VectorCopyRow/VectorAssignRow/VectorDeserializeRow assign needs explicit
isNull[offset] modification (Gopal V, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/branch-1 b625586f6 -> 3734d5b67
Vectorization: VectorCopyRow/VectorAssignRow/VectorDeserializeRow assign needs explicit isNull[offset] modification (Gopal V, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3734d5b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3734d5b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3734d5b6
Branch: refs/heads/branch-1
Commit: 3734d5b674b4e8de9c0cc751650aee3194bfb93a
Parents: b625586
Author: Matt McCline <mm...@hortonworks.com>
Authored: Fri Apr 29 16:36:00 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Fri Apr 29 16:36:00 2016 -0700
----------------------------------------------------------------------
.../hive/ql/exec/vector/VectorAssignRow.java | 12 ++++++++++++
.../hadoop/hive/ql/exec/vector/VectorCopyRow.java | 10 ++++++++++
.../hive/ql/exec/vector/VectorDeserializeRow.java | 18 ++++++++++++++++++
.../hive/ql/exec/vector/TestVectorRowObject.java | 7 +++++++
.../hive/ql/exec/vector/TestVectorSerDeRow.java | 10 ++++++++++
5 files changed, 57 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3734d5b6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index 9502134..70e2e46 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -138,6 +138,7 @@ public abstract class VectorAssignRow {
} else {
BooleanWritable bw = (BooleanWritable) object;
vector[batchIndex] = (bw.get() ? 1 : 0);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -155,6 +156,7 @@ public abstract class VectorAssignRow {
} else {
ByteWritable bw = (ByteWritable) object;
vector[batchIndex] = bw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -172,6 +174,7 @@ public abstract class VectorAssignRow {
} else {
ShortWritable sw = (ShortWritable) object;
vector[batchIndex] = sw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -189,6 +192,7 @@ public abstract class VectorAssignRow {
} else {
IntWritable iw = (IntWritable) object;
vector[batchIndex] = iw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -206,6 +210,7 @@ public abstract class VectorAssignRow {
} else {
LongWritable lw = (LongWritable) object;
vector[batchIndex] = lw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -223,6 +228,7 @@ public abstract class VectorAssignRow {
} else {
DateWritable bw = (DateWritable) object;
vector[batchIndex] = bw.getDays();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -373,6 +379,7 @@ public abstract class VectorAssignRow {
} else {
DoubleWritable dw = (DoubleWritable) object;
vector[batchIndex] = dw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -409,6 +416,7 @@ public abstract class VectorAssignRow {
} else {
BytesWritable bw = (BytesWritable) object;
colVector.setVal(batchIndex, bw.getBytes(), 0, bw.getLength());
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -426,6 +434,7 @@ public abstract class VectorAssignRow {
} else {
Text tw = (Text) object;
colVector.setVal(batchIndex, tw.getBytes(), 0, tw.getLength());
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -450,6 +459,7 @@ public abstract class VectorAssignRow {
}
byte[] bytes = hiveVarchar.getValue().getBytes();
colVector.setVal(batchIndex, bytes, 0, bytes.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -476,6 +486,7 @@ public abstract class VectorAssignRow {
// We store CHAR in vector row batch with padding stripped.
byte[] bytes = hiveChar.getStrippedValue().getBytes();
colVector.setVal(batchIndex, bytes, 0, bytes.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -508,6 +519,7 @@ public abstract class VectorAssignRow {
} else {
colVector.set(batchIndex, (HiveDecimalWritable) object);
}
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3734d5b6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index 97b1f24..85ec80b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -57,12 +57,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.vector[outBatchIndex] = inColVector.vector[0];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -84,12 +86,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.vector[outBatchIndex] = inColVector.vector[0];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex];
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -119,12 +123,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.setVal(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.setVal(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -146,12 +152,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
outColVector.setRef(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
outColVector.setRef(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]);
+ outColVector.isNull[outBatchIndex] = false;
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
@@ -172,12 +180,14 @@ public class VectorCopyRow {
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.set(outBatchIndex, inColVector.vector[0]);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
}
} else {
if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.isNull[outBatchIndex] = false;
outColVector.set(outBatchIndex, inColVector.vector[inBatchIndex]);
} else {
VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
http://git-wip-us.apache.org/repos/asf/hive/blob/3734d5b6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index 0ff5083..5f5fbe2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -100,6 +100,7 @@ public class VectorDeserializeRow {
} else {
boolean value = deserializeRead.readBoolean();
colVector.vector[batchIndex] = (value ? 1 : 0);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -119,6 +120,7 @@ public class VectorDeserializeRow {
} else {
byte value = deserializeRead.readByte();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -138,6 +140,7 @@ public class VectorDeserializeRow {
} else {
short value = deserializeRead.readShort();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -157,6 +160,7 @@ public class VectorDeserializeRow {
} else {
int value = deserializeRead.readInt();
colVector.vector[batchIndex] = (long) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -176,6 +180,7 @@ public class VectorDeserializeRow {
} else {
long value = deserializeRead.readLong();
colVector.vector[batchIndex] = value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -198,6 +203,7 @@ public class VectorDeserializeRow {
} else {
deserializeRead.readDate(readDateResults);
colVector.vector[batchIndex] = (long) readDateResults.getDays();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -252,6 +258,7 @@ public class VectorDeserializeRow {
deserializeRead.readIntervalYearMonth(readIntervalYearMonthResults);
HiveIntervalYearMonth hiym = readIntervalYearMonthResults.getHiveIntervalYearMonth();
colVector.vector[batchIndex] = hiym.getTotalMonths();
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -309,6 +316,7 @@ public class VectorDeserializeRow {
} else {
float value = deserializeRead.readFloat();
colVector.vector[batchIndex] = (double) value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -328,6 +336,7 @@ public class VectorDeserializeRow {
} else {
double value = deserializeRead.readDouble();
colVector.vector[batchIndex] = value;
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -358,6 +367,7 @@ public class VectorDeserializeRow {
deserializeRead.readString(readStringResults);
colVector.setVal(batchIndex, readStringResults.bytes,
readStringResults.start, readStringResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -381,6 +391,7 @@ public class VectorDeserializeRow {
deserializeRead.readString(readStringResults);
colVector.setRef(batchIndex, readStringResults.bytes,
readStringResults.start, readStringResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -410,6 +421,7 @@ public class VectorDeserializeRow {
int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, charTypeInfo.getLength());
colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -439,6 +451,7 @@ public class VectorDeserializeRow {
int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, charTypeInfo.getLength());
colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -468,6 +481,7 @@ public class VectorDeserializeRow {
int adjustedLength = StringExpr.truncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, varcharTypeInfo.getLength());
colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -497,6 +511,7 @@ public class VectorDeserializeRow {
int adjustedLength = StringExpr.truncate(readStringResults.bytes,
readStringResults.start, readStringResults.length, varcharTypeInfo.getLength());
colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -520,6 +535,7 @@ public class VectorDeserializeRow {
deserializeRead.readBinary(readBinaryResults);
colVector.setVal(batchIndex, readBinaryResults.bytes,
readBinaryResults.start, readBinaryResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -543,6 +559,7 @@ public class VectorDeserializeRow {
deserializeRead.readBinary(readBinaryResults);
colVector.setRef(batchIndex, readBinaryResults.bytes,
readBinaryResults.start, readBinaryResults.length);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -566,6 +583,7 @@ public class VectorDeserializeRow {
deserializeRead.readHiveDecimal(readDecimalResults);
HiveDecimal hiveDecimal = readDecimalResults.getHiveDecimal();
colVector.vector[batchIndex].set(hiveDecimal);
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3734d5b6/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
index c076e6c..a5946d1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -60,6 +61,12 @@ public class TestVectorRowObject extends TestCase {
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+ // junk the destination for the 1st pass
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ cv.noNulls = false;
+ }
+
VectorAssignRowSameBatch vectorAssignRow = new VectorAssignRowSameBatch();
vectorAssignRow.init(source.typeNames());
vectorAssignRow.setOneBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/3734d5b6/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
index d3dc30d..e62e78c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
@@ -573,6 +573,10 @@ public class TestVectorSerDeRow extends TestCase {
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ }
+
int fieldCount = source.typeNames().size();
DeserializeRead deserializeRead;
SerializeWrite serializeWrite;
@@ -602,6 +606,12 @@ public class TestVectorSerDeRow extends TestCase {
VectorDeserializeRow vectorDeserializeRow = new VectorDeserializeRow(deserializeRead);
vectorDeserializeRow.init();
+ // junk the destination for the 1st pass
+ for (ColumnVector cv : batch.cols) {
+ Arrays.fill(cv.isNull, true);
+ cv.noNulls = false;
+ }
+
VectorExtractRowSameBatch vectorExtractRow = new VectorExtractRowSameBatch();
vectorExtractRow.init(source.typeNames());
vectorExtractRow.setOneBatch(batch);