You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/11/09 17:30:44 UTC
svn commit: r1540340 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/vector/
test/org/apache/hadoop/hive/ql/exec/vector/
test/org/apache/hadoop/hive/ql/exec/vector/expressions/
Author: hashutosh
Date: Sat Nov 9 16:30:44 2013
New Revision: 1540340
URL: http://svn.apache.org/r1540340
Log:
HIVE-5699 : Add unit test for vectorized BETWEEN for timestamp inputs (Eric Hanson via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1540340&r1=1540339&r2=1540340&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Sat Nov 9 16:30:44 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.Ex
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode;
import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
@@ -557,7 +558,7 @@ public class VectorizationContext {
List<ExprNodeDesc> childExpr, Mode mode) throws HiveException {
//First handle special cases
if (udf instanceof GenericUDFBetween) {
- return getBetweenFilterExpression(childExpr);
+ return getBetweenFilterExpression(childExpr, mode);
} else if (udf instanceof GenericUDFIn) {
return getInFilterExpression(childExpr);
} else if (udf instanceof GenericUDFBridge) {
@@ -757,9 +758,16 @@ public class VectorizationContext {
* needs to be done differently than the standard way where all arguments are
* passed to the VectorExpression constructor.
*/
- private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr)
+ private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr, Mode mode)
throws HiveException {
+ if (mode == Mode.PROJECTION) {
+
+ // Projection mode is not yet supported for [NOT] BETWEEN. Return null so Vectorizer
+ // knows to revert to row-at-a-time execution.
+ return null;
+ }
+
boolean notKeywordPresent = (Boolean) ((ExprNodeConstantDesc) childExpr.get(0)).getValue();
ExprNodeDesc colExpr = childExpr.get(1);
@@ -1000,10 +1008,7 @@ public class VectorizationContext {
throw new HiveException("Udf: failed to convert to timestamp");
}
Timestamp ts = (Timestamp) java;
- long result = ts.getTime();
- result *= 1000000; // shift left 6 digits to make room for nanos below ms precision
- result += ts.getNanos() % 1000000; // add in nanos, after removing the ms portion
- return result;
+ return TimestampUtils.getTimeNanoSec(ts);
}
private Constructor<?> getConstructor(Class<?> cl) throws HiveException {
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1540340&r1=1540339&r2=1540340&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Sat Nov 9 16:30:44 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.v
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -942,6 +943,19 @@ public class TestVectorizationContext {
children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterDoubleColumnNotBetween);
+
+ // timestamp BETWEEN
+ children1.set(0, new ExprNodeConstantDesc(new Boolean(false)));
+ children1.set(1, new ExprNodeColumnDesc(Timestamp.class, "col1", "table", false));
+ children1.set(2, new ExprNodeConstantDesc("2013-11-05 00:00:00.000"));
+ children1.set(3, new ExprNodeConstantDesc("2013-11-06 00:00:00.000"));
+ ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
+ assertTrue(ve instanceof FilterLongColumnBetween);
+
+ // timestamp NOT BETWEEN
+ children1.set(0, new ExprNodeConstantDesc(new Boolean(true)));
+ ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
+ assertTrue(ve instanceof FilterLongColumnNotBetween);
}
@Test
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java?rev=1540340&r1=1540339&r2=1540340&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java Sat Nov 9 16:30:44 2013
@@ -22,9 +22,12 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.sql.Timestamp;
+
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween;
@@ -544,6 +547,59 @@ public class TestVectorFilterExpressions
assertEquals(0, vrb.selected[0]);
}
+ @Test
+ public void testFilterTimestampBetween() {
+ int seed = 17;
+ VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch(
+ 5, 2, seed);
+ LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0];
+ long startTS = 0; // the epoch
+ long endTS = TimestampUtils.getTimeNanoSec(
+ Timestamp.valueOf("2013-11-05 00:00:00.000000000"));
+
+ Timestamp ts0 = Timestamp.valueOf("1963-11-06 00:00:00.000");
+ lcv0.vector[0] = TimestampUtils.getTimeNanoSec(ts0);
+ Timestamp ts1 = Timestamp.valueOf("1983-11-06 00:00:00.000");
+ lcv0.vector[1] = TimestampUtils.getTimeNanoSec(ts1);
+ Timestamp ts2 = Timestamp.valueOf("2099-11-06 00:00:00.000");
+ lcv0.vector[2] = TimestampUtils.getTimeNanoSec(ts2);
+ vrb.size = 3;
+
+ VectorExpression expr1 = new FilterLongColumnBetween(0, startTS, endTS);
+ expr1.evaluate(vrb);
+ assertEquals(1, vrb.size);
+ assertEquals(true, vrb.selectedInUse);
+ assertEquals(1, vrb.selected[0]);
+ }
+
+ @Test
+ public void testFilterTimestampNotBetween() {
+ int seed = 17;
+ VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch(
+ 5, 2, seed);
+ LongColumnVector lcv0 = (LongColumnVector) vrb.cols[0];
+ long startTS = TimestampUtils.getTimeNanoSec(
+ Timestamp.valueOf("2013-11-05 00:00:00.000000000"));
+ long endTS = TimestampUtils.getTimeNanoSec(
+ Timestamp.valueOf("2013-11-05 00:00:00.000000010"));
+
+ Timestamp ts0 = Timestamp.valueOf("2013-11-04 00:00:00.000000000");
+ lcv0.vector[0] = TimestampUtils.getTimeNanoSec(ts0);
+ Timestamp ts1 = Timestamp.valueOf("2013-11-05 00:00:00.000000002");
+ lcv0.vector[1] = TimestampUtils.getTimeNanoSec(ts1);
+ Timestamp ts2 = Timestamp.valueOf("2099-11-06 00:00:00.000");
+ lcv0.vector[2] = TimestampUtils.getTimeNanoSec(ts2);
+ vrb.size = 3;
+
+ VectorExpression expr1 = new FilterLongColumnNotBetween(0, startTS, endTS);
+ expr1.evaluate(vrb);
+ assertEquals(2, vrb.size);
+ assertEquals(true, vrb.selectedInUse);
+ assertEquals(0, vrb.selected[0]);
+ assertEquals(2, vrb.selected[1]);
+
+ }
+
/**
* Test the IN filter VectorExpression classes.
*/