You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/04/19 12:13:07 UTC
[13/20] hive git commit: HIVE-9862 Vectorized execution corrupts
timestamp values (Matt McCline,
reviewed by Jason Dere) HIVE-13111: Fix timestamp / interval_day_time wrong
results with HIVE-9862 (Matt McCline, reviewed by Jason Dere)
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
new file mode 100644
index 0000000..c0dd5ed
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.sql.Timestamp;
+import java.util.Arrays;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents a nullable timestamp column vector capable of handing a wide range of
+ * timestamp values.
+ *
+ * We store the 2 (value) fields of a Timestamp class in primitive arrays.
+ *
+ * We do this to avoid an array of Java Timestamp objects which would have poor storage
+ * and memory access characteristics.
+ *
+ * Generally, the caller will fill in a scratch timestamp object with values from a row, work
+ * using the scratch timestamp, and then perhaps update the column vector row with a result.
+ */
+public class TimestampColumnVector extends ColumnVector {
+
+ /*
+ * The storage arrays for this column vector corresponds to the storage of a Timestamp:
+ */
+ public long[] time;
+ // The values from Timestamp.getTime().
+
+ public int[] nanos;
+ // The values from Timestamp.getNanos().
+
+ /*
+ * Scratch objects.
+ */
+ private final Timestamp scratchTimestamp;
+
+ private Writable scratchWritable;
+ // Supports keeping a TimestampWritable object without having to import that definition...
+
+ /**
+ * Use this constructor by default. All column vectors
+ * should normally be the default size.
+ */
+ public TimestampColumnVector() {
+ this(VectorizedRowBatch.DEFAULT_SIZE);
+ }
+
+ /**
+ * Don't use this except for testing purposes.
+ *
+ * @param len the number of rows
+ */
+ public TimestampColumnVector(int len) {
+ super(len);
+
+ time = new long[len];
+ nanos = new int[len];
+
+ scratchTimestamp = new Timestamp(0);
+
+ scratchWritable = null; // Allocated by caller.
+ }
+
+ /**
+ * Return the number of rows.
+ * @return
+ */
+ public int getLength() {
+ return time.length;
+ }
+
+ /**
+ * Return a row's Timestamp.getTime() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public long getTime(int elementNum) {
+ return time[elementNum];
+ }
+
+ /**
+ * Return a row's Timestamp.getNanos() value.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public int getNanos(int elementNum) {
+ return nanos[elementNum];
+ }
+
+ /**
+ * Set a Timestamp object from a row of the column.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param timestamp
+ * @param elementNum
+ */
+ public void timestampUpdate(Timestamp timestamp, int elementNum) {
+ timestamp.setTime(time[elementNum]);
+ timestamp.setNanos(nanos[elementNum]);
+ }
+
+ /**
+ * Return the scratch Timestamp object set from a row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @return
+ */
+ public Timestamp asScratchTimestamp(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return scratchTimestamp;
+ }
+
+ /**
+ * Return the scratch timestamp (contents undefined).
+ * @return
+ */
+ public Timestamp getScratchTimestamp() {
+ return scratchTimestamp;
+ }
+
+ /**
+ * Return a long representation of a Timestamp.
+ * @param elementNum
+ * @return
+ */
+ public long getTimestampAsLong(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getTimestampAsLong(scratchTimestamp);
+ }
+
+ /**
+ * Return a long representation of a Timestamp.
+ * @param timestamp
+ * @return
+ */
+ public static long getTimestampAsLong(Timestamp timestamp) {
+ return millisToSeconds(timestamp.getTime());
+ }
+
+ // Copy of TimestampWritable.millisToSeconds
+ /**
+ * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of
+ * seconds. 500 would round to 0, -500 would round to -1.
+ */
+ private static long millisToSeconds(long millis) {
+ if (millis >= 0) {
+ return millis / 1000;
+ } else {
+ return (millis - 999) / 1000;
+ }
+ }
+
+ /**
+ * Return a double representation of a Timestamp.
+ * @param elementNum
+ * @return
+ */
+ public double getDouble(int elementNum) {
+ scratchTimestamp.setTime(time[elementNum]);
+ scratchTimestamp.setNanos(nanos[elementNum]);
+ return getDouble(scratchTimestamp);
+ }
+
+ /**
+ * Return a double representation of a Timestamp.
+ * @param elementNum
+ * @return
+ */
+ public static double getDouble(Timestamp timestamp) {
+ // Same algorithm as TimestampWritable (not currently import-able here).
+ double seconds, nanos;
+ seconds = millisToSeconds(timestamp.getTime());
+ nanos = timestamp.getNanos();
+ return seconds + nanos / 1000000000;
+ }
+
+ /**
+ * Compare row to Timestamp.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param elementNum
+ * @param timestamp
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(int elementNum, Timestamp timestamp) {
+ return asScratchTimestamp(elementNum).compareTo(timestamp);
+ }
+
+ /**
+ * Compare Timestamp to row.
+ * We assume the entry has already been NULL checked and isRepeated adjusted.
+ * @param timestamp
+ * @param elementNum
+ * @return -1, 0, 1 standard compareTo values.
+ */
+ public int compareTo(Timestamp timestamp, int elementNum) {
+ return timestamp.compareTo(asScratchTimestamp(elementNum));
+ }
+
+ /**
+ * Compare a row to another TimestampColumnVector's row.
+ * @param elementNum1
+ * @param timestampColVector2
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(int elementNum1, TimestampColumnVector timestampColVector2,
+ int elementNum2) {
+ return asScratchTimestamp(elementNum1).compareTo(
+ timestampColVector2.asScratchTimestamp(elementNum2));
+ }
+
+ /**
+ * Compare another TimestampColumnVector's row to a row.
+ * @param timestampColVector1
+ * @param elementNum1
+ * @param elementNum2
+ * @return
+ */
+ public int compareTo(TimestampColumnVector timestampColVector1, int elementNum1,
+ int elementNum2) {
+ return timestampColVector1.asScratchTimestamp(elementNum1).compareTo(
+ asScratchTimestamp(elementNum2));
+ }
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+
+ TimestampColumnVector timestampColVector = (TimestampColumnVector) inputVector;
+
+ time[outElementNum] = timestampColVector.time[inputElementNum];
+ nanos[outElementNum] = timestampColVector.nanos[inputElementNum];
+ }
+
+ // Simplify vector by brute-force flattening noNulls and isRepeating
+ // This can be used to reduce combinatorial explosion of code paths in VectorExpressions
+ // with many arguments.
+ public void flatten(boolean selectedInUse, int[] sel, int size) {
+ flattenPush();
+ if (isRepeating) {
+ isRepeating = false;
+ long repeatFastTime = time[0];
+ int repeatNanos = nanos[0];
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ time[i] = repeatFastTime;
+ nanos[i] = repeatNanos;
+ }
+ } else {
+ Arrays.fill(time, 0, size, repeatFastTime);
+ Arrays.fill(nanos, 0, size, repeatNanos);
+ }
+ flattenRepeatingNulls(selectedInUse, sel, size);
+ }
+ flattenNoNulls(selectedInUse, sel, size);
+ }
+
+ /**
+ * Set a row from a timestamp.
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ * @param timestamp
+ */
+ public void set(int elementNum, Timestamp timestamp) {
+ this.time[elementNum] = timestamp.getTime();
+ this.nanos[elementNum] = timestamp.getNanos();
+ }
+
+ /**
+ * Set a row from the current value in the scratch timestamp.
+ * @param elementNum
+ */
+ public void setFromScratchTimestamp(int elementNum) {
+ this.time[elementNum] = scratchTimestamp.getTime();
+ this.nanos[elementNum] = scratchTimestamp.getNanos();
+ }
+
+ /**
+ * Set row to standard null value(s).
+ * We assume the entry has already been isRepeated adjusted.
+ * @param elementNum
+ */
+ public void setNullValue(int elementNum) {
+ time[elementNum] = 0;
+ nanos[elementNum] = 1;
+ }
+
+ // Copy the current object contents into the output. Only copy selected entries,
+ // as indicated by selectedInUse and the sel array.
+ public void copySelected(
+ boolean selectedInUse, int[] sel, int size, TimestampColumnVector output) {
+
+ // Output has nulls if and only if input has nulls.
+ output.noNulls = noNulls;
+ output.isRepeating = false;
+
+ // Handle repeating case
+ if (isRepeating) {
+ output.time[0] = time[0];
+ output.nanos[0] = nanos[0];
+ output.isNull[0] = isNull[0];
+ output.isRepeating = true;
+ return;
+ }
+
+ // Handle normal case
+
+ // Copy data values over
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.time[i] = time[i];
+ output.nanos[i] = nanos[i];
+ }
+ }
+ else {
+ System.arraycopy(time, 0, output.time, 0, size);
+ System.arraycopy(nanos, 0, output.nanos, 0, size);
+ }
+
+ // Copy nulls over if needed
+ if (!noNulls) {
+ if (selectedInUse) {
+ for (int j = 0; j < size; j++) {
+ int i = sel[j];
+ output.isNull[i] = isNull[i];
+ }
+ }
+ else {
+ System.arraycopy(isNull, 0, output.isNull, 0, size);
+ }
+ }
+ }
+
+ /**
+ * Fill all the vector entries with a timestamp.
+ * @param timestamp
+ */
+ public void fill(Timestamp timestamp) {
+ noNulls = true;
+ isRepeating = true;
+ time[0] = timestamp.getTime();
+ nanos[0] = timestamp.getNanos();
+ }
+
+ /**
+ * Return a convenience writable object stored by this column vector.
+ * Supports keeping a TimestampWritable object without having to import that definition...
+ * @return
+ */
+ public Writable getScratchWritable() {
+ return scratchWritable;
+ }
+
+ /**
+ * Set the convenience writable object stored by this column vector
+ * @param scratchWritable
+ */
+ public void setScratchWritable(Writable scratchWritable) {
+ this.scratchWritable = scratchWritable;
+ }
+
+ @Override
+ public void stringifyValue(StringBuilder buffer, int row) {
+ if (isRepeating) {
+ row = 0;
+ }
+ if (noNulls || !isNull[row]) {
+ scratchTimestamp.setTime(time[row]);
+ scratchTimestamp.setNanos(nanos[row]);
+ buffer.append(scratchTimestamp.toString());
+ } else {
+ buffer.append("null");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
index 95dbf8d..bb795fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
@@ -18,50 +18,32 @@
package org.apache.hadoop.hive.ql.exec.vector;
-import java.sql.Timestamp;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
public final class TimestampUtils {
- /**
- * Store the given timestamp in nanoseconds into the timestamp object.
- * @param timeInNanoSec Given timestamp in nanoseconds
- * @param t The timestamp object
- */
- public static void assignTimeInNanoSec(long timeInNanoSec, Timestamp t) {
- /*
- * java.sql.Timestamp consists of a long variable to store milliseconds and an integer variable for nanoseconds.
- * The long variable is used to store only the full seconds converted to millis. For example for 1234 milliseconds,
- * 1000 is stored in the long variable, and 234000000 (234 converted to nanoseconds) is stored as nanoseconds.
- * The negative timestamps are also supported, but nanoseconds must be positive therefore millisecond part is
- * reduced by one second.
- */
- long integralSecInMillis = (timeInNanoSec / 1000000000) * 1000; // Full seconds converted to millis.
- long nanos = timeInNanoSec % 1000000000; // The nanoseconds.
- if (nanos < 0) {
- nanos = 1000000000 + nanos; // The positive nano-part that will be added to milliseconds.
- integralSecInMillis = ((timeInNanoSec / 1000000000) - 1) * 1000; // Reduce by one second.
- }
- t.setTime(integralSecInMillis);
- t.setNanos((int) nanos);
- }
-
- public static long getTimeNanoSec(Timestamp t) {
- long time = t.getTime();
- int nanos = t.getNanos();
- return (time * 1000000) + (nanos % 1000000);
- }
+ static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
+ static final long NANOSECONDS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
- public static long secondsToNanoseconds(long seconds) {
- return seconds * 1000000000;
+ public static long daysToNanoseconds(long daysSinceEpoch) {
+ return DateWritable.daysToMillis((int) daysSinceEpoch) * NANOSECONDS_PER_MILLISECOND;
}
- public static long doubleToNanoseconds(double d) {
- return (long) (d * 1000000000);
+ public static TimestampWritable timestampColumnVectorWritable(
+ TimestampColumnVector timestampColVector, int elementNum,
+ TimestampWritable timestampWritable) {
+ timestampWritable.set(timestampColVector.asScratchTimestamp(elementNum));
+ return timestampWritable;
}
- public static long daysToNanoseconds(long daysSinceEpoch) {
- return DateWritable.daysToMillis((int) daysSinceEpoch) * 1000000;
+ public static HiveIntervalDayTimeWritable intervalDayTimeColumnVectorWritable(
+ IntervalDayTimeColumnVector intervalDayTimeColVector, int elementNum,
+ HiveIntervalDayTimeWritable intervalDayTimeWritable) {
+ intervalDayTimeWritable.set(intervalDayTimeColVector.asScratchIntervalDayTime(elementNum));
+ return intervalDayTimeWritable;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index 809d7d4..9502134 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec.vector;
-import java.sql.Timestamp;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -228,7 +227,26 @@ public abstract class VectorAssignRow {
}
}
- private class TimestampAssigner extends AbstractLongAssigner {
+ private abstract class AbstractTimestampAssigner extends Assigner {
+
+ protected TimestampColumnVector colVector;
+
+ AbstractTimestampAssigner(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (TimestampColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class TimestampAssigner extends AbstractTimestampAssigner {
TimestampAssigner(int columnIndex) {
super(columnIndex);
@@ -239,9 +257,8 @@ public abstract class VectorAssignRow {
if (object == null) {
VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex);
} else {
- TimestampWritable tw = (TimestampWritable) object;
- Timestamp t = tw.getTimestamp();
- vector[batchIndex] = TimestampUtils.getTimeNanoSec(t);
+ colVector.set(batchIndex, ((TimestampWritable) object).getTimestamp());
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -260,11 +277,31 @@ public abstract class VectorAssignRow {
HiveIntervalYearMonthWritable iymw = (HiveIntervalYearMonthWritable) object;
HiveIntervalYearMonth iym = iymw.getHiveIntervalYearMonth();
vector[batchIndex] = iym.getTotalMonths();
+ colVector.isNull[batchIndex] = false;
}
}
}
- private class IntervalDayTimeAssigner extends AbstractLongAssigner {
+ private abstract class AbstractIntervalDayTimeAssigner extends Assigner {
+
+ protected IntervalDayTimeColumnVector colVector;
+
+ AbstractIntervalDayTimeAssigner(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class IntervalDayTimeAssigner extends AbstractIntervalDayTimeAssigner {
IntervalDayTimeAssigner(int columnIndex) {
super(columnIndex);
@@ -277,7 +314,8 @@ public abstract class VectorAssignRow {
} else {
HiveIntervalDayTimeWritable idtw = (HiveIntervalDayTimeWritable) object;
HiveIntervalDayTime idt = idtw.getHiveIntervalDayTime();
- vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(idt);
+ colVector.set(batchIndex, idt);
+ colVector.isNull[batchIndex] = false;
}
}
}
@@ -317,6 +355,7 @@ public abstract class VectorAssignRow {
} else {
FloatWritable fw = (FloatWritable) object;
vector[batchIndex] = fw.get();
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
index befe2fc..96b8f78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
@@ -165,6 +166,27 @@ public class VectorColumnAssignFactory {
}
}
+ private static abstract class VectorTimestampColumnAssign
+ extends VectorColumnAssignVectorBase<TimestampColumnVector> {
+
+ protected void assignTimestamp(Timestamp value, int index) {
+ outCol.set(index, value);
+ }
+ protected void assignTimestamp(TimestampWritable tw, int index) {
+ outCol.set(index, tw.getTimestamp());
+ }
+ }
+
+ private static abstract class VectorIntervalDayTimeColumnAssign
+ extends VectorColumnAssignVectorBase<IntervalDayTimeColumnVector> {
+
+ protected void assignIntervalDayTime(HiveIntervalDayTime value, int index) {
+ outCol.set(index, value);
+ }
+ protected void assignIntervalDayTime(HiveIntervalDayTimeWritable tw, int index) {
+ outCol.set(index, tw.getHiveIntervalDayTime());
+ }
+ }
public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch)
throws HiveException {
@@ -313,19 +335,17 @@ public class VectorColumnAssignFactory {
}.init(outputBatch, (LongColumnVector) destCol);
break;
case TIMESTAMP:
- outVCA = new VectorLongColumnAssign() {
+ outVCA = new VectorTimestampColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
assignNull(destIndex);
}
else {
- TimestampWritable bw = (TimestampWritable) val;
- Timestamp t = bw.getTimestamp();
- assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
+ assignTimestamp((TimestampWritable) val, destIndex);
}
}
- }.init(outputBatch, (LongColumnVector) destCol);
+ }.init(outputBatch, (TimestampColumnVector) destCol);
break;
case DATE:
outVCA = new VectorLongColumnAssign() {
@@ -355,7 +375,7 @@ public class VectorColumnAssignFactory {
}
}.init(outputBatch, (LongColumnVector) destCol);
break;
- case INTERVAL_DAY_TIME:outVCA = new VectorLongColumnAssign() {
+ case INTERVAL_DAY_TIME:outVCA = new VectorIntervalDayTimeColumnAssign() {
@Override
public void assignObjectValue(Object val, int destIndex) throws HiveException {
if (val == null) {
@@ -363,12 +383,12 @@ public class VectorColumnAssignFactory {
}
else {
HiveIntervalDayTimeWritable bw = (HiveIntervalDayTimeWritable) val;
- assignLong(
- DateUtils.getIntervalDayTimeTotalNanos(bw.getHiveIntervalDayTime()),
+ assignIntervalDayTime(
+ bw.getHiveIntervalDayTime(),
destIndex);
}
}
- }.init(outputBatch, (LongColumnVector) destCol);
+ }.init(outputBatch, (IntervalDayTimeColumnVector) destCol);
break;
default:
throw new HiveException("Incompatible Long vector column and primitive category " +
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
index 8c4b6ea..935b47b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.util.Arrays;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* Class to keep information on a set of typed vector columns. Used by
@@ -52,6 +55,16 @@ public class VectorColumnSetInfo {
protected int[] decimalIndices;
/**
+ * indices of TIMESTAMP primitive keys.
+ */
+ protected int[] timestampIndices;
+
+ /**
+ * indices of INTERVAL_DAY_TIME primitive keys.
+ */
+ protected int[] intervalDayTimeIndices;
+
+ /**
* Helper class for looking up a key value based on key index.
*/
public class KeyLookupHelper {
@@ -59,11 +72,14 @@ public class VectorColumnSetInfo {
public int doubleIndex;
public int stringIndex;
public int decimalIndex;
+ public int timestampIndex;
+ public int intervalDayTimeIndex;
private static final int INDEX_UNUSED = -1;
private void resetIndices() {
- this.longIndex = this.doubleIndex = this.stringIndex = this.decimalIndex = INDEX_UNUSED;
+ this.longIndex = this.doubleIndex = this.stringIndex = this.decimalIndex =
+ timestampIndex = intervalDayTimeIndex = INDEX_UNUSED;
}
public void setLong(int index) {
resetIndices();
@@ -84,6 +100,16 @@ public class VectorColumnSetInfo {
resetIndices();
this.decimalIndex = index;
}
+
+ public void setTimestamp(int index) {
+ resetIndices();
+ this.timestampIndex= index;
+ }
+
+ public void setIntervalDayTime(int index) {
+ resetIndices();
+ this.intervalDayTimeIndex= index;
+ }
}
/**
@@ -98,6 +124,8 @@ public class VectorColumnSetInfo {
protected int doubleIndicesIndex;
protected int stringIndicesIndex;
protected int decimalIndicesIndex;
+ protected int timestampIndicesIndex;
+ protected int intervalDayTimeIndicesIndex;
protected VectorColumnSetInfo(int keyCount) {
this.keyCount = keyCount;
@@ -112,33 +140,62 @@ public class VectorColumnSetInfo {
stringIndicesIndex = 0;
decimalIndices = new int[this.keyCount];
decimalIndicesIndex = 0;
+ timestampIndices = new int[this.keyCount];
+ timestampIndicesIndex = 0;
+ intervalDayTimeIndices = new int[this.keyCount];
+ intervalDayTimeIndicesIndex = 0;
indexLookup = new KeyLookupHelper[this.keyCount];
}
protected void addKey(String outputType) throws HiveException {
indexLookup[addIndex] = new KeyLookupHelper();
- if (VectorizationContext.isIntFamily(outputType) ||
- VectorizationContext.isDatetimeFamily(outputType)) {
+
+ String typeName = VectorizationContext.mapTypeNameSynonyms(outputType);
+
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
+
+ switch (columnVectorType) {
+ case LONG:
longIndices[longIndicesIndex] = addIndex;
indexLookup[addIndex].setLong(longIndicesIndex);
++longIndicesIndex;
- } else if (VectorizationContext.isFloatFamily(outputType)) {
+ break;
+
+ case DOUBLE:
doubleIndices[doubleIndicesIndex] = addIndex;
indexLookup[addIndex].setDouble(doubleIndicesIndex);
++doubleIndicesIndex;
- } else if (VectorizationContext.isStringFamily(outputType) ||
- outputType.equalsIgnoreCase("binary")) {
+ break;
+
+ case BYTES:
stringIndices[stringIndicesIndex]= addIndex;
indexLookup[addIndex].setString(stringIndicesIndex);
++stringIndicesIndex;
- } else if (VectorizationContext.isDecimalFamily(outputType)) {
- decimalIndices[decimalIndicesIndex]= addIndex;
- indexLookup[addIndex].setDecimal(decimalIndicesIndex);
- ++decimalIndicesIndex;
- }
- else {
- throw new HiveException("Unsuported vector output type: " + outputType);
+ break;
+
+ case DECIMAL:
+ decimalIndices[decimalIndicesIndex]= addIndex;
+ indexLookup[addIndex].setDecimal(decimalIndicesIndex);
+ ++decimalIndicesIndex;
+ break;
+
+ case TIMESTAMP:
+ timestampIndices[timestampIndicesIndex] = addIndex;
+ indexLookup[addIndex].setTimestamp(timestampIndicesIndex);
+ ++timestampIndicesIndex;
+ break;
+
+ case INTERVAL_DAY_TIME:
+ intervalDayTimeIndices[intervalDayTimeIndicesIndex] = addIndex;
+ indexLookup[addIndex].setIntervalDayTime(intervalDayTimeIndicesIndex);
+ ++intervalDayTimeIndicesIndex;
+ break;
+
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
+
addIndex++;
}
@@ -147,5 +204,7 @@ public class VectorColumnSetInfo {
doubleIndices = Arrays.copyOf(doubleIndices, doubleIndicesIndex);
stringIndices = Arrays.copyOf(stringIndices, stringIndicesIndex);
decimalIndices = Arrays.copyOf(decimalIndices, decimalIndicesIndex);
+ timestampIndices = Arrays.copyOf(timestampIndices, timestampIndicesIndex);
+ intervalDayTimeIndices = Arrays.copyOf(intervalDayTimeIndices, intervalDayTimeIndicesIndex);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index 0058141..97b1f24 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* This class copies specified columns of a row from one VectorizedRowBatch to another.
*/
@@ -183,10 +186,64 @@ public class VectorCopyRow {
}
}
+ private class TimestampCopyRow extends CopyRow {
+
+ TimestampCopyRow(int inColumnIndex, int outColumnIndex) {
+ super(inColumnIndex, outColumnIndex);
+ }
+
+ @Override
+ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBatch, int outBatchIndex) {
+ TimestampColumnVector inColVector = (TimestampColumnVector) inBatch.cols[inColumnIndex];
+ TimestampColumnVector outColVector = (TimestampColumnVector) outBatch.cols[outColumnIndex];
+
+ if (inColVector.isRepeating) {
+ if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.setElement(outBatchIndex, 0, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ } else {
+ if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.setElement(outBatchIndex, inBatchIndex, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ }
+ }
+ }
+
+ private class IntervalDayTimeCopyRow extends CopyRow {
+
+ IntervalDayTimeCopyRow(int inColumnIndex, int outColumnIndex) {
+ super(inColumnIndex, outColumnIndex);
+ }
+
+ @Override
+ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBatch, int outBatchIndex) {
+ IntervalDayTimeColumnVector inColVector = (IntervalDayTimeColumnVector) inBatch.cols[inColumnIndex];
+ IntervalDayTimeColumnVector outColVector = (IntervalDayTimeColumnVector) outBatch.cols[outColumnIndex];
+
+ if (inColVector.isRepeating) {
+ if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.setElement(outBatchIndex, 0, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ } else {
+ if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.setElement(outBatchIndex, inBatchIndex, inColVector);
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ }
+ }
+ }
+
private CopyRow[] subRowToBatchCopiersByValue;
private CopyRow[] subRowToBatchCopiersByReference;
- public void init(VectorColumnMapping columnMapping) {
+ public void init(VectorColumnMapping columnMapping) throws HiveException {
int count = columnMapping.getCount();
subRowToBatchCopiersByValue = new CopyRow[count];
subRowToBatchCopiersByReference = new CopyRow[count];
@@ -194,24 +251,43 @@ public class VectorCopyRow {
for (int i = 0; i < count; i++) {
int inputColumn = columnMapping.getInputColumns()[i];
int outputColumn = columnMapping.getOutputColumns()[i];
- String typeName = columnMapping.getTypeNames()[i];
+ String typeName = columnMapping.getTypeNames()[i].toLowerCase();
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName);
+ Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
CopyRow copyRowByValue = null;
CopyRow copyRowByReference = null;
- if (VectorizationContext.isIntFamily(typeName) ||
- VectorizationContext.isDatetimeFamily(typeName)) {
+ switch (columnVectorType) {
+ case LONG:
copyRowByValue = new LongCopyRow(inputColumn, outputColumn);
- } else if (VectorizationContext.isFloatFamily(typeName)) {
+ break;
+
+ case TIMESTAMP:
+ copyRowByValue = new TimestampCopyRow(inputColumn, outputColumn);
+ break;
+
+ case INTERVAL_DAY_TIME:
+ copyRowByValue = new IntervalDayTimeCopyRow(inputColumn, outputColumn);
+ break;
+
+ case DOUBLE:
copyRowByValue = new DoubleCopyRow(inputColumn, outputColumn);
- } else if (VectorizationContext.isStringFamily(typeName)) {
+ break;
+
+ case BYTES:
copyRowByValue = new BytesCopyRowByValue(inputColumn, outputColumn);
copyRowByReference = new BytesCopyRowByReference(inputColumn, outputColumn);
- } else if (VectorizationContext.decimalTypePattern.matcher(typeName).matches()){
+ break;
+
+ case DECIMAL:
copyRowByValue = new DecimalCopyRow(inputColumn, outputColumn);
- } else {
- throw new RuntimeException("Cannot allocate vector copy row for " + typeName);
+ break;
+
+ default:
+ throw new HiveException("Unexpected column vector type " + columnVectorType);
}
+
subRowToBatchCopiersByValue[i] = copyRowByValue;
if (copyRowByReference == null) {
subRowToBatchCopiersByReference[i] = copyRowByValue;
@@ -243,4 +319,19 @@ public class VectorCopyRow {
copyRow.copy(inBatch, inBatchIndex, outBatch, outBatchIndex);
}
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("VectorCopyRow ");
+ for (CopyRow copyRow : subRowToBatchCopiersByValue) {
+ if (sb.length() > 0) {
+ sb.append(", ");
+ }
+ sb.append(copyRow.getClass().getName());
+ sb.append(" inColumnIndex " + copyRow.inColumnIndex);
+ sb.append(" outColumnIndex " + copyRow.outColumnIndex);
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index 56cf9ba..0ff5083 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.io.EOFException;
import java.io.IOException;
-import java.sql.Timestamp;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -203,7 +202,14 @@ public class VectorDeserializeRow {
}
}
- private class TimestampReader extends AbstractLongReader {
+ private abstract class AbstractTimestampReader extends Reader {
+
+ AbstractTimestampReader(int columnIndex) {
+ super(columnIndex);
+ }
+ }
+
+ private class TimestampReader extends AbstractTimestampReader {
DeserializeRead.ReadTimestampResults readTimestampResults;
@@ -214,16 +220,17 @@ public class VectorDeserializeRow {
@Override
void apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
+ TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex];
if (deserializeRead.readCheckNull()) {
VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex);
} else {
deserializeRead.readTimestamp(readTimestampResults);
- Timestamp t = readTimestampResults.getTimestamp();
- colVector.vector[batchIndex] = TimestampUtils.getTimeNanoSec(t);
+ colVector.set(batchIndex, readTimestampResults.getTimestamp());
+ colVector.isNull[batchIndex] = false;
}
}
+
}
private class IntervalYearMonthReader extends AbstractLongReader {
@@ -249,7 +256,14 @@ public class VectorDeserializeRow {
}
}
- private class IntervalDayTimeReader extends AbstractLongReader {
+ private abstract class AbstractIntervalDayTimeReader extends Reader {
+
+ AbstractIntervalDayTimeReader(int columnIndex) {
+ super(columnIndex);
+ }
+ }
+
+ private class IntervalDayTimeReader extends AbstractIntervalDayTimeReader {
DeserializeRead.ReadIntervalDayTimeResults readIntervalDayTimeResults;
@@ -260,14 +274,15 @@ public class VectorDeserializeRow {
@Override
void apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
+ IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
if (deserializeRead.readCheckNull()) {
VectorizedBatchUtil.setNullColIsNullValue(colVector, batchIndex);
} else {
deserializeRead.readIntervalDayTime(readIntervalDayTimeResults);
- HiveIntervalDayTime hidt = readIntervalDayTimeResults.getHiveIntervalDayTime();
- colVector.vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(hidt);
+ HiveIntervalDayTime idt = readIntervalDayTimeResults.getHiveIntervalDayTime();
+ colVector.set(batchIndex, idt);
+ colVector.isNull[batchIndex] = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
index 9d241bd..b018979 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
@@ -43,7 +43,7 @@ public class VectorExpressionDescriptor {
// LongColumnVector -->
// INT_FAMILY
// DATE
- // TIMESTAMP
+ // INTERVAL_FAMILY
//
// DoubleColumnVector -->
// FLOAT_FAMILY
@@ -56,6 +56,12 @@ public class VectorExpressionDescriptor {
// CHAR
// VARCHAR
//
+ // TimestampColumnVector -->
+ // TIMESTAMP
+ //
+ // IntervalDayTimeColumnVector -->
+ // INTERVAL_DAY_TIME
+ //
public enum ArgumentType {
NONE (0x000),
INT_FAMILY (0x001),
@@ -71,9 +77,8 @@ public class VectorExpressionDescriptor {
INTERVAL_DAY_TIME (0x200),
DATETIME_FAMILY (DATE.value | TIMESTAMP.value),
INTERVAL_FAMILY (INTERVAL_YEAR_MONTH.value | INTERVAL_DAY_TIME.value),
- INT_TIMESTAMP_FAMILY (INT_FAMILY.value | TIMESTAMP.value),
- INT_INTERVAL_FAMILY (INT_FAMILY.value | INTERVAL_FAMILY.value),
- INT_DATETIME_INTERVAL_FAMILY (INT_FAMILY.value | DATETIME_FAMILY.value | INTERVAL_FAMILY.value),
+ INT_INTERVAL_YEAR_MONTH (INT_FAMILY.value | INTERVAL_YEAR_MONTH.value),
+ INT_DATE_INTERVAL_YEAR_MONTH (INT_FAMILY.value | DATE.value | INTERVAL_YEAR_MONTH.value),
STRING_DATETIME_FAMILY (STRING_FAMILY.value | DATETIME_FAMILY.value),
ALL_FAMILY (0xFFF);
@@ -146,10 +151,13 @@ public class VectorExpressionDescriptor {
public static String getVectorColumnSimpleName(ArgumentType argType) {
if (argType == INT_FAMILY ||
argType == DATE ||
- argType == TIMESTAMP ||
- argType == INTERVAL_YEAR_MONTH ||
- argType == INTERVAL_DAY_TIME) {
+ argType == INTERVAL_YEAR_MONTH
+ ) {
return "Long";
+ } else if (argType == TIMESTAMP) {
+ return "Timestamp";
+ } else if (argType == INTERVAL_DAY_TIME) {
+ return "IntervalDayTime";
} else if (argType == FLOAT_FAMILY) {
return "Double";
} else if (argType == DECIMAL) {
@@ -341,7 +349,7 @@ public class VectorExpressionDescriptor {
return ve;
}
} catch (Exception ex) {
- throw new HiveException(ex);
+ throw new HiveException("Could not instantiate VectorExpression class " + ve.getSimpleName(), ex);
}
}
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index c662634..2a9a179 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -257,10 +257,29 @@ public abstract class VectorExtractRow {
}
}
- private class TimestampExtractor extends AbstractLongExtractor {
+ private abstract class AbstractTimestampExtractor extends Extractor {
+
+ protected TimestampColumnVector colVector;
+
+ AbstractTimestampExtractor(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (TimestampColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class TimestampExtractor extends AbstractTimestampExtractor {
+
+ protected Timestamp timestamp;
- private Timestamp timestamp;
-
TimestampExtractor(int columnIndex) {
super(columnIndex);
object = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector.create(new Timestamp(0));
@@ -271,8 +290,7 @@ public abstract class VectorExtractRow {
Object extract(int batchIndex) {
int adjustedIndex = (colVector.isRepeating ? 0 : batchIndex);
if (colVector.noNulls || !colVector.isNull[adjustedIndex]) {
- long value = vector[adjustedIndex];
- TimestampUtils.assignTimeInNanoSec(value, timestamp);
+ colVector.timestampUpdate(timestamp, adjustedIndex);
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector.set(object, timestamp);
return object;
} else {
@@ -284,7 +302,7 @@ public abstract class VectorExtractRow {
private class IntervalYearMonthExtractor extends AbstractLongExtractor {
private HiveIntervalYearMonth hiveIntervalYearMonth;
-
+
IntervalYearMonthExtractor(int columnIndex) {
super(columnIndex);
object = PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector.create(new HiveIntervalYearMonth(0));
@@ -305,10 +323,29 @@ public abstract class VectorExtractRow {
}
}
- private class IntervalDayTimeExtractor extends AbstractLongExtractor {
+ private abstract class AbstractIntervalDayTimeExtractor extends Extractor {
+
+ protected IntervalDayTimeColumnVector colVector;
+
+ AbstractIntervalDayTimeExtractor(int columnIndex) {
+ super(columnIndex);
+ }
+
+ @Override
+ void setColumnVector(VectorizedRowBatch batch) {
+ colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ }
+
+ @Override
+ void forgetColumnVector() {
+ colVector = null;
+ }
+ }
+
+ private class IntervalDayTimeExtractor extends AbstractIntervalDayTimeExtractor {
private HiveIntervalDayTime hiveIntervalDayTime;
-
+
IntervalDayTimeExtractor(int columnIndex) {
super(columnIndex);
object = PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector.create(new HiveIntervalDayTime(0, 0));
@@ -319,8 +356,7 @@ public abstract class VectorExtractRow {
Object extract(int batchIndex) {
int adjustedIndex = (colVector.isRepeating ? 0 : batchIndex);
if (colVector.noNulls || !colVector.isNull[adjustedIndex]) {
- long value = vector[adjustedIndex];
- DateUtils.setIntervalDayTimeTotalNanos(hiveIntervalDayTime, value);
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(adjustedIndex));
PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector.set(object, hiveIntervalDayTime);
return object;
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
index fabac38..50d0452 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.io.IOException;
-import java.util.Arrays;
-
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -121,5 +119,29 @@ public class VectorGroupKeyHelper extends VectorColumnSetInfo {
outputColumnVector.isNull[outputBatch.size] = true;
}
}
+ for(int i=0;i<timestampIndices.length; ++i) {
+ int keyIndex = timestampIndices[i];
+ TimestampColumnVector inputColumnVector = (TimestampColumnVector) inputBatch.cols[keyIndex];
+ TimestampColumnVector outputColumnVector = (TimestampColumnVector) outputBatch.cols[keyIndex];
+ if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) {
+
+ outputColumnVector.setElement(outputBatch.size, 0, inputColumnVector);
+ } else {
+ outputColumnVector.noNulls = false;
+ outputColumnVector.isNull[outputBatch.size] = true;
+ }
+ }
+ for(int i=0;i<intervalDayTimeIndices.length; ++i) {
+ int keyIndex = intervalDayTimeIndices[i];
+ IntervalDayTimeColumnVector inputColumnVector = (IntervalDayTimeColumnVector) inputBatch.cols[keyIndex];
+ IntervalDayTimeColumnVector outputColumnVector = (IntervalDayTimeColumnVector) outputBatch.cols[keyIndex];
+ if (inputColumnVector.noNulls || !inputColumnVector.isNull[0]) {
+
+ outputColumnVector.setElement(outputBatch.size, 0, inputColumnVector);
+ } else {
+ outputColumnVector.noNulls = false;
+ outputColumnVector.isNull[outputBatch.size] = true;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
index aff3551..8a101a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
@@ -18,14 +18,16 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import java.sql.Timestamp;
import java.util.Arrays;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.exec.KeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
@@ -42,6 +44,8 @@ public class VectorHashKeyWrapper extends KeyWrapper {
private static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
private static final byte[][] EMPTY_BYTES_ARRAY = new byte[0][];
private static final HiveDecimalWritable[] EMPTY_DECIMAL_ARRAY = new HiveDecimalWritable[0];
+ private static final Timestamp[] EMPTY_TIMESTAMP_ARRAY = new Timestamp[0];
+ private static final HiveIntervalDayTime[] EMPTY_INTERVAL_DAY_TIME_ARRAY = new HiveIntervalDayTime[0];
private long[] longValues;
private double[] doubleValues;
@@ -52,14 +56,21 @@ public class VectorHashKeyWrapper extends KeyWrapper {
private HiveDecimalWritable[] decimalValues;
+ private Timestamp[] timestampValues;
+
+ private HiveIntervalDayTime[] intervalDayTimeValues;
+
private boolean[] isNull;
private int hashcode;
public VectorHashKeyWrapper(int longValuesCount, int doubleValuesCount,
- int byteValuesCount, int decimalValuesCount) {
+ int byteValuesCount, int decimalValuesCount, int timestampValuesCount,
+ int intervalDayTimeValuesCount) {
longValues = longValuesCount > 0 ? new long[longValuesCount] : EMPTY_LONG_ARRAY;
doubleValues = doubleValuesCount > 0 ? new double[doubleValuesCount] : EMPTY_DOUBLE_ARRAY;
decimalValues = decimalValuesCount > 0 ? new HiveDecimalWritable[decimalValuesCount] : EMPTY_DECIMAL_ARRAY;
+ timestampValues = timestampValuesCount > 0 ? new Timestamp[timestampValuesCount] : EMPTY_TIMESTAMP_ARRAY;
+ intervalDayTimeValues = intervalDayTimeValuesCount > 0 ? new HiveIntervalDayTime[intervalDayTimeValuesCount] : EMPTY_INTERVAL_DAY_TIME_ARRAY;
for(int i = 0; i < decimalValuesCount; ++i) {
decimalValues[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
}
@@ -72,7 +83,14 @@ public class VectorHashKeyWrapper extends KeyWrapper {
byteStarts = EMPTY_INT_ARRAY;
byteLengths = EMPTY_INT_ARRAY;
}
- isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount + decimalValuesCount];
+ for(int i = 0; i < timestampValuesCount; ++i) {
+ timestampValues[i] = new Timestamp(0);
+ }
+ for(int i = 0; i < intervalDayTimeValuesCount; ++i) {
+ intervalDayTimeValues[i] = new HiveIntervalDayTime();
+ }
+ isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount +
+ decimalValuesCount + timestampValuesCount + intervalDayTimeValuesCount];
hashcode = 0;
}
@@ -94,6 +112,14 @@ public class VectorHashKeyWrapper extends KeyWrapper {
hashcode ^= decimalValues[i].getHiveDecimal().hashCode();
}
+ for (int i = 0; i < timestampValues.length; i++) {
+ hashcode ^= timestampValues[i].hashCode();
+ }
+
+ for (int i = 0; i < intervalDayTimeValues.length; i++) {
+ hashcode ^= intervalDayTimeValues[i].hashCode();
+ }
+
// This code, with branches and all, is not executed if there are no string keys
for (int i = 0; i < byteValues.length; ++i) {
/*
@@ -131,6 +157,8 @@ public class VectorHashKeyWrapper extends KeyWrapper {
Arrays.equals(longValues, keyThat.longValues) &&
Arrays.equals(doubleValues, keyThat.doubleValues) &&
Arrays.equals(decimalValues, keyThat.decimalValues) &&
+ Arrays.equals(timestampValues, keyThat.timestampValues) &&
+ Arrays.equals(intervalDayTimeValues, keyThat.intervalDayTimeValues) &&
Arrays.equals(isNull, keyThat.isNull) &&
byteValues.length == keyThat.byteValues.length &&
(0 == byteValues.length || bytesEquals(keyThat));
@@ -196,6 +224,23 @@ public class VectorHashKeyWrapper extends KeyWrapper {
clone.byteStarts = EMPTY_INT_ARRAY;
clone.byteLengths = EMPTY_INT_ARRAY;
}
+ if (timestampValues.length > 0) {
+ clone.timestampValues = new Timestamp[timestampValues.length];
+ for(int i = 0; i < timestampValues.length; ++i) {
+ clone.timestampValues[i] = (Timestamp) timestampValues[i].clone();
+ }
+ } else {
+ clone.timestampValues = EMPTY_TIMESTAMP_ARRAY;
+ }
+ if (intervalDayTimeValues.length > 0) {
+ clone.intervalDayTimeValues = new HiveIntervalDayTime[intervalDayTimeValues.length];
+ for(int i = 0; i < intervalDayTimeValues.length; ++i) {
+ clone.intervalDayTimeValues[i] = (HiveIntervalDayTime) intervalDayTimeValues[i].clone();
+ }
+ } else {
+ clone.intervalDayTimeValues = EMPTY_INTERVAL_DAY_TIME_ARRAY;
+ }
+
clone.hashcode = hashcode;
assert clone.equals(this);
}
@@ -256,14 +301,50 @@ public class VectorHashKeyWrapper extends KeyWrapper {
isNull[longValues.length + doubleValues.length + byteValues.length + index] = true;
}
+ public void assignTimestamp(int index, Timestamp value) {
+ timestampValues[index] = value;
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + index] = false;
+ }
+
+ public void assignTimestamp(int index, TimestampColumnVector colVector, int elementNum) {
+ colVector.timestampUpdate(timestampValues[index], elementNum);
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + index] = false;
+ }
+
+ public void assignNullTimestamp(int index) {
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + index] = true;
+ }
+
+ public void assignIntervalDayTime(int index, HiveIntervalDayTime value) {
+ intervalDayTimeValues[index].set(value);
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = false;
+ }
+
+ public void assignIntervalDayTime(int index, IntervalDayTimeColumnVector colVector, int elementNum) {
+ intervalDayTimeValues[index].set(colVector.asScratchIntervalDayTime(elementNum));
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = false;
+ }
+
+ public void assignNullIntervalDayTime(int index) {
+ isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + index] = true;
+ }
+
@Override
public String toString()
{
- return String.format("%d[%s] %d[%s] %d[%s] %d[%s]",
+ return String.format("%d[%s] %d[%s] %d[%s] %d[%s] %d[%s] %d[%s]",
longValues.length, Arrays.toString(longValues),
doubleValues.length, Arrays.toString(doubleValues),
byteValues.length, Arrays.toString(byteValues),
- decimalValues.length, Arrays.toString(decimalValues));
+ decimalValues.length, Arrays.toString(decimalValues),
+ timestampValues.length, Arrays.toString(timestampValues),
+ intervalDayTimeValues.length, Arrays.toString(intervalDayTimeValues));
}
public boolean getIsLongNull(int i) {
@@ -315,5 +396,23 @@ public class VectorHashKeyWrapper extends KeyWrapper {
public HiveDecimalWritable getDecimal(int i) {
return decimalValues[i];
}
+
+ public boolean getIsTimestampNull(int i) {
+ return isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + i];
+ }
+
+ public Timestamp getTimestamp(int i) {
+ return timestampValues[i];
+ }
+
+ public boolean getIsIntervalDayTimeNull(int i) {
+ return isNull[longValues.length + doubleValues.length + byteValues.length +
+ decimalValues.length + timestampValues.length + i];
+ }
+
+ public HiveIntervalDayTime getIntervalDayTime(int i) {
+ return intervalDayTimeValues[i];
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index 6333222..bfd26ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.hive.ql.exec.vector;
-import java.util.Arrays;
-
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
/**
* Class for handling vectorized hash map key wrappers. It evaluates the key columns in a
@@ -157,27 +155,71 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
}
for(int i=0;i<decimalIndices.length; ++i) {
- int keyIndex = decimalIndices[i];
- int columnIndex = keyExpressions[keyIndex].getOutputColumn();
- DecimalColumnVector columnVector = (DecimalColumnVector) batch.cols[columnIndex];
- if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
- } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
- } else if (columnVector.noNulls && columnVector.isRepeating) {
- assignDecimalNoNullsRepeating(i, batch.size, columnVector);
- } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
- assignDecimalNullsNoRepeatingNoSelection(i, batch.size, columnVector);
- } else if (!columnVector.noNulls && columnVector.isRepeating) {
- assignDecimalNullsRepeating(i, batch.size, columnVector);
- } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
- assignDecimalNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
- } else {
- throw new HiveException (String.format(
- "Unimplemented Decimal null/repeat/selected combination %b/%b/%b",
- columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
- }
+ int keyIndex = decimalIndices[i];
+ int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ DecimalColumnVector columnVector = (DecimalColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented Decimal null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<timestampIndices.length; ++i) {
+ int keyIndex = timestampIndices[i];
+ int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ TimestampColumnVector columnVector = (TimestampColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignTimestampNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignTimestampNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignTimestampNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignTimestampNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignTimestampNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignTimestampNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented timestamp null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
+ for(int i=0;i<intervalDayTimeIndices.length; ++i) {
+ int keyIndex = intervalDayTimeIndices[i];
+ int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ IntervalDayTimeColumnVector columnVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignIntervalDayTimeNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignIntervalDayTimeNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented intervalDayTime null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
}
+ }
for(int i=0;i<batch.size;++i) {
vectorHashKeyWrappers[i].setHashKey();
}
@@ -504,6 +546,154 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
}
/**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, possible nulls, no repeat values, batch selection vector.
+ */
+ private void assignTimestampNullsNoRepeatingSelection(int index, int size,
+ TimestampColumnVector columnVector, int[] selected) {
+ for(int i = 0; i < size; ++i) {
+ int row = selected[i];
+ if (!columnVector.isNull[row]) {
+ vectorHashKeyWrappers[i].assignTimestamp(index, columnVector, row);
+ } else {
+ vectorHashKeyWrappers[i].assignNullTimestamp(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, repeat null values.
+ */
+ private void assignTimestampNullsRepeating(int index, int size,
+ TimestampColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignNullTimestamp(index);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, possible nulls, repeat values.
+ */
+ private void assignTimestampNullsNoRepeatingNoSelection(int index, int size,
+ TimestampColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ if (!columnVector.isNull[r]) {
+ vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, r);
+ } else {
+ vectorHashKeyWrappers[r].assignNullTimestamp(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, no nulls, repeat values, no selection vector.
+ */
+ private void assignTimestampNoNullsRepeating(int index, int size, TimestampColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, 0);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, no nulls, no repeat values, batch selection vector.
+ */
+ private void assignTimestampNoNullsNoRepeatingSelection(int index, int size,
+ TimestampColumnVector columnVector, int[] selected) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, selected[r]);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Timestamp type, no nulls, no repeat values, no selection vector.
+ */
+ private void assignTimestampNoNullsNoRepeatingNoSelection(int index, int size,
+ TimestampColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignTimestamp(index, columnVector, r);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, possible nulls, no repeat values, batch selection vector.
+ */
+ private void assignIntervalDayTimeNullsNoRepeatingSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector, int[] selected) {
+ for(int i = 0; i < size; ++i) {
+ int row = selected[i];
+ if (!columnVector.isNull[row]) {
+ vectorHashKeyWrappers[i].assignIntervalDayTime(index, columnVector, row);
+ } else {
+ vectorHashKeyWrappers[i].assignNullIntervalDayTime(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, repeat null values.
+ */
+ private void assignIntervalDayTimeNullsRepeating(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, possible nulls, repeat values.
+ */
+ private void assignIntervalDayTimeNullsNoRepeatingNoSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ if (!columnVector.isNull[r]) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, r);
+ } else {
+ vectorHashKeyWrappers[r].assignNullIntervalDayTime(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, repeat values, no selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsRepeating(int index, int size, IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, 0);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, no repeat values, batch selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsNoRepeatingSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector, int[] selected) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, selected[r]);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for IntervalDayTime type, no nulls, no repeat values, no selection vector.
+ */
+ private void assignIntervalDayTimeNoNullsNoRepeatingNoSelection(int index, int size,
+ IntervalDayTimeColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignIntervalDayTime(index, columnVector, r);
+ }
+ }
+
+ /**
* Prepares a VectorHashKeyWrapperBatch to work for a specific set of keys.
* Computes the fast access lookup indices, preallocates all needed internal arrays.
* This step is done only once per query, not once per batch. The information computed now
@@ -544,16 +734,19 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(compiledKeyWrapperBatch.doubleIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.stringIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.decimalIndices.length);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.timestampIndices.length);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(compiledKeyWrapperBatch.intervalDayTimeIndices.length);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForIntArrayOfSize(compiledKeyWrapperBatch.longIndices.length) * 2;
compiledKeyWrapperBatch.keysFixedSize +=
model.lengthForBooleanArrayOfSize(keyExpressions.length);
return compiledKeyWrapperBatch;
}
-
+
public VectorHashKeyWrapper allocateKeyWrapper() {
return new VectorHashKeyWrapper(longIndices.length, doubleIndices.length,
- stringIndices.length, decimalIndices.length);
+ stringIndices.length, decimalIndices.length, timestampIndices.length,
+ intervalDayTimeIndices.length);
}
/**
@@ -581,11 +774,19 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
return kw.getIsDecimalNull(klh.decimalIndex)? null :
keyOutputWriter.writeValue(
kw.getDecimal(klh.decimalIndex).getHiveDecimal());
- }
- else {
+ } else if (klh.timestampIndex >= 0) {
+ return kw.getIsTimestampNull(klh.timestampIndex)? null :
+ keyOutputWriter.writeValue(
+ kw.getTimestamp(klh.timestampIndex));
+ } else if (klh.intervalDayTimeIndex >= 0) {
+ return kw.getIsIntervalDayTimeNull(klh.intervalDayTimeIndex)? null :
+ keyOutputWriter.writeValue(
+ kw.getIntervalDayTime(klh.intervalDayTimeIndex));
+ } else {
throw new HiveException(String.format(
- "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d",
- i, klh.longIndex, klh.doubleIndex, klh.stringIndex, klh.decimalIndex));
+ "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d %d %d %d",
+ i, klh.longIndex, klh.doubleIndex, klh.stringIndex, klh.decimalIndex,
+ klh.timestampIndex, klh.intervalDayTimeIndex));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
index 5586944..47acbe4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.sql.Timestamp;
import java.util.List;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -254,7 +255,7 @@ public class VectorSerializeRow {
}
}
- private class TimestampWriter extends AbstractLongWriter {
+ private class TimestampWriter extends Writer {
Timestamp scratchTimestamp;
@@ -265,11 +266,11 @@ public class VectorSerializeRow {
@Override
boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
+ TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex];
if (colVector.isRepeating) {
if (colVector.noNulls || !colVector.isNull[0]) {
- TimestampUtils.assignTimeInNanoSec(colVector.vector[0], scratchTimestamp);
+ colVector.timestampUpdate(scratchTimestamp, 0);
serializeWrite.writeTimestamp(scratchTimestamp);
return true;
} else {
@@ -278,7 +279,7 @@ public class VectorSerializeRow {
}
} else {
if (colVector.noNulls || !colVector.isNull[batchIndex]) {
- TimestampUtils.assignTimeInNanoSec(colVector.vector[batchIndex], scratchTimestamp);
+ colVector.timestampUpdate(scratchTimestamp, batchIndex);
serializeWrite.writeTimestamp(scratchTimestamp);
return true;
} else {
@@ -319,19 +320,23 @@ public class VectorSerializeRow {
}
}
- private class IntervalDayTimeWriter extends AbstractLongWriter {
+ private class IntervalDayTimeWriter extends Writer {
+
+ private HiveIntervalDayTime hiveIntervalDayTime;
IntervalDayTimeWriter(int columnIndex) {
super(columnIndex);
+ hiveIntervalDayTime = new HiveIntervalDayTime();
}
@Override
boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
+ IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
if (colVector.isRepeating) {
if (colVector.noNulls || !colVector.isNull[0]) {
- serializeWrite.writeHiveIntervalDayTime(colVector.vector[0]);
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(0));
+ serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime);
return true;
} else {
serializeWrite.writeNull();
@@ -339,7 +344,8 @@ public class VectorSerializeRow {
}
} else {
if (colVector.noNulls || !colVector.isNull[batchIndex]) {
- serializeWrite.writeHiveIntervalDayTime(colVector.vector[batchIndex]);
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(batchIndex));
+ serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime);
return true;
} else {
serializeWrite.writeNull();
http://git-wip-us.apache.org/repos/asf/hive/blob/130293e5/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRowNoNulls.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRowNoNulls.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRowNoNulls.java
index 1363004..6b5740e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRowNoNulls.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRowNoNulls.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
-import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -170,8 +169,8 @@ public class VectorSerializeRowNoNulls {
@Override
void apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
- TimestampUtils.assignTimeInNanoSec(colVector.vector[colVector.isRepeating ? 0 : batchIndex], scratchTimestamp);
+ TimestampColumnVector colVector = (TimestampColumnVector) batch.cols[columnIndex];
+ colVector.timestampUpdate(scratchTimestamp, colVector.isRepeating ? 0 : batchIndex);
serializeWrite.writeTimestamp(scratchTimestamp);
}
}
@@ -191,14 +190,18 @@ public class VectorSerializeRowNoNulls {
private class IntervalDayTimeWriter extends AbstractLongWriter {
+ private HiveIntervalDayTime hiveIntervalDayTime;
+
IntervalDayTimeWriter(int columnIndex) {
super(columnIndex);
+ hiveIntervalDayTime = new HiveIntervalDayTime();
}
@Override
void apply(VectorizedRowBatch batch, int batchIndex) throws IOException {
- LongColumnVector colVector = (LongColumnVector) batch.cols[columnIndex];
- serializeWrite.writeHiveIntervalDayTime(colVector.vector[colVector.isRepeating ? 0 : batchIndex]);
+ IntervalDayTimeColumnVector colVector = (IntervalDayTimeColumnVector) batch.cols[columnIndex];
+ hiveIntervalDayTime.set(colVector.asScratchIntervalDayTime(colVector.isRepeating ? 0 : batchIndex));
+ serializeWrite.writeHiveIntervalDayTime(hiveIntervalDayTime);
}
}