You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/07/19 19:22:36 UTC

svn commit: r1611926 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/vector/ java/org/apache/hadoop/hive/ql/exec/vector/expressions/ java/org/apache/hadoop/hive/ql/optimizer/physical/ test/queries/clientpositive/ test/results/clientpositive/

Author: hashutosh
Date: Sat Jul 19 17:22:36 2014
New Revision: 1611926

URL: http://svn.apache.org/r1611926
Log:
HIVE-7054 : Support ELT UDF in vectorized mode (Deepesh Khandelwal via Jitendra Nath Pandey)

Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
    hive/trunk/ql/src/test/queries/clientpositive/vector_elt.q
    hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1611926&r1=1611925&r2=1611926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Sat Jul 19 17:22:36 2014
@@ -100,6 +100,7 @@ import org.apache.hadoop.hive.serde2.typ
 import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 
 /**
  * Context class for vectorization execution.
@@ -393,13 +394,30 @@ public class VectorizationContext {
 
     List<ExprNodeDesc> childrenWithCasts = new ArrayList<ExprNodeDesc>();
     boolean atleastOneCastNeeded = false;
-    for (ExprNodeDesc child : children) {
-      ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
-      if (castExpression != null) {
-        atleastOneCastNeeded = true;
-        childrenWithCasts.add(castExpression);
-      } else {
-        childrenWithCasts.add(child);
+    if (genericUDF instanceof GenericUDFElt) {
+      int i = 0;
+      for (ExprNodeDesc child : children) {
+        TypeInfo castType = commonType;
+        if (i++ == 0) {
+          castType = isIntFamily(child.getTypeString()) ? child.getTypeInfo() : TypeInfoFactory.intTypeInfo;
+        }
+        ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, castType);
+        if (castExpression != null) {
+          atleastOneCastNeeded = true;
+          childrenWithCasts.add(castExpression);
+        } else {
+          childrenWithCasts.add(child);
+        }
+      }
+    } else {
+      for (ExprNodeDesc child : children) {
+        ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
+        if (castExpression != null) {
+          atleastOneCastNeeded = true;
+          childrenWithCasts.add(castExpression);
+        } else {
+          childrenWithCasts.add(child);
+        }
       }
     }
     if (atleastOneCastNeeded) {
@@ -484,7 +502,7 @@ public class VectorizationContext {
     } else {
 
       // Casts to exact types including long to double etc. are needed in some special cases.
-      if (udf instanceof GenericUDFCoalesce) {
+      if (udf instanceof GenericUDFCoalesce || udf instanceof GenericUDFElt) {
         GenericUDF genericUdf = getGenericUDFForCast(castType);
         List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
         children.add(child);
@@ -896,6 +914,10 @@ public class VectorizationContext {
 
       // Coalesce is a special case because it can take variable number of arguments.
       return getCoalesceExpression(childExpr, returnType);
+    } else if (udf instanceof GenericUDFElt) {
+
+      // Coalesce is a special case because it can take variable number of arguments.
+      return getEltExpression(childExpr, returnType);
     } else if (udf instanceof GenericUDFBridge) {
       VectorExpression v = getGenericUDFBridgeVectorExpression((GenericUDFBridge) udf, childExpr, mode,
           returnType);
@@ -948,6 +970,33 @@ public class VectorizationContext {
     }
   }
 
+  private VectorExpression getEltExpression(List<ExprNodeDesc> childExpr, TypeInfo returnType)
+      throws HiveException {
+    int[] inputColumns = new int[childExpr.size()];
+    VectorExpression[] vectorChildren = null;
+    try {
+      vectorChildren = getVectorExpressions(childExpr, Mode.PROJECTION);
+
+      int i = 0;
+      for (VectorExpression ve : vectorChildren) {
+        inputColumns[i++] = ve.getOutputColumn();
+      }
+
+      int outColumn = ocm.allocateOutputColumn(getNormalizedTypeName(returnType.getTypeName()));
+      VectorElt vectorElt = new VectorElt(inputColumns, outColumn);
+      vectorElt.setOutputType(returnType.getTypeName());
+      vectorElt.setChildExpressions(vectorChildren);
+      return vectorElt;
+    } finally {
+      // Free the output columns of the child expressions.
+      if (vectorChildren != null) {
+        for (VectorExpression v : vectorChildren) {
+          ocm.freeOutputColumn(v.getOutputColumn());
+        }
+      }
+    }
+  }
+
   /**
    * Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
    */

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java?rev=1611926&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java Sat Jul 19 17:22:36 2014
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+public class VectorElt extends VectorExpression {
+
+  private static final long serialVersionUID = 1L;
+  private int [] inputColumns;
+  private int outputColumn;
+
+  public VectorElt(int [] inputColumns, int outputColumn) {
+    this();
+    this.inputColumns = inputColumns;
+    this.outputColumn = outputColumn;
+  }
+
+  public VectorElt() {
+    super();
+  }
+
+  @Override
+  public void evaluate(VectorizedRowBatch batch) {
+
+    if (childExpressions != null) {
+      super.evaluateChildren(batch);
+    }
+
+    int[] sel = batch.selected;
+    int n = batch.size;
+    BytesColumnVector outputVector = (BytesColumnVector) batch.cols[outputColumn];
+    if (n <= 0) {
+      return;
+    }
+
+    outputVector.init();
+
+    outputVector.noNulls = false;
+    outputVector.isRepeating = false;
+
+    LongColumnVector inputIndexVector = (LongColumnVector) batch.cols[inputColumns[0]];
+    long[] indexVector = inputIndexVector.vector;
+    if (inputIndexVector.isRepeating) {
+      int index = (int)indexVector[0];
+      if (index > 0 && index < inputColumns.length) {
+        BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+        if (cv.isRepeating) {
+          outputVector.setElement(0, 0, cv);
+          outputVector.isRepeating = true;
+        } else if (batch.selectedInUse) {
+          for (int j = 0; j != n; j++) {
+            int i = sel[j];
+            outputVector.setVal(i, cv.vector[0], cv.start[0], cv.length[0]);
+          }
+        } else {
+          for (int i = 0; i != n; i++) {
+            outputVector.setVal(i, cv.vector[0], cv.start[0], cv.length[0]);
+          }
+        }
+      } else {
+        outputVector.isNull[0] = true;
+        outputVector.isRepeating = true;
+      }
+    } else if (batch.selectedInUse) {
+      for (int j = 0; j != n; j++) {
+        int i = sel[j];
+        int index = (int)indexVector[i];
+        if (index > 0 && index < inputColumns.length) {
+          BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+          int cvi = cv.isRepeating ? 0 : i;
+          outputVector.setVal(i, cv.vector[cvi], cv.start[cvi], cv.length[cvi]);
+        } else {
+          outputVector.isNull[i] = true;
+        }
+      }
+    } else {
+      for (int i = 0; i != n; i++) {
+        int index = (int)indexVector[i];
+        if (index > 0 && index < inputColumns.length) {
+          BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+          int cvi = cv.isRepeating ? 0 : i;
+          outputVector.setVal(i, cv.vector[cvi], cv.start[cvi], cv.length[cvi]);
+        } else {
+          outputVector.isNull[i] = true;
+        }
+      }
+    }
+  }
+
+  @Override
+  public int getOutputColumn() {
+    return outputColumn;
+  }
+
+  @Override
+  public String getOutputType() {
+    return outputType;
+  }
+
+  public int [] getInputColumns() {
+    return inputColumns;
+  }
+
+  public void setInputColumns(int [] inputColumns) {
+    this.inputColumns = inputColumns;
+  }
+
+  public void setOutputColumn(int outputColumn) {
+    this.outputColumn = outputColumn;
+  }
+
+  @Override
+  public VectorExpressionDescriptor.Descriptor getDescriptor() {
+    // Descriptor is not defined because it takes variable number of arguments with different
+    // data types.
+    throw new UnsupportedOperationException("Undefined descriptor");
+  }
+}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1611926&r1=1611925&r2=1611926&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Sat Jul 19 17:22:36 2014
@@ -222,6 +222,7 @@ public class Vectorizer implements Physi
     supportedGenericUDFs.add(GenericUDFCase.class);
     supportedGenericUDFs.add(GenericUDFWhen.class);
     supportedGenericUDFs.add(GenericUDFCoalesce.class);
+    supportedGenericUDFs.add(GenericUDFElt.class);
 
     // For type casts
     supportedGenericUDFs.add(UDFToLong.class);

Added: hive/trunk/ql/src/test/queries/clientpositive/vector_elt.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_elt.q?rev=1611926&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_elt.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_elt.q Sat Jul 19 17:22:36 2014
@@ -0,0 +1,34 @@
+SET hive.vectorized.execution.enabled=true;
+
+EXPLAIN SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10;
+
+SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10;
+
+EXPLAIN
+SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1;
+
+SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1;

Added: hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out?rev=1611926&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_elt.q.out Sat Jul 19 17:22:36 2014
@@ -0,0 +1,154 @@
+PREHOOK: query: EXPLAIN SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: alltypesorc
+            Statistics: Num rows: 3492 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (ctinyint > 0) (type: boolean)
+              Statistics: Num rows: 1164 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                expressions: ((ctinyint % 2) + 1) (type: int), cstring1 (type: string), cint (type: int), elt(((ctinyint % 2) + 1), cstring1, cint) (type: string)
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 1164 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
+                Limit
+                  Number of rows: 10
+                  Statistics: Num rows: 10 Data size: 1080 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 10 Data size: 1080 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+      Execution mode: vectorized
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 10
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT (ctinyint % 2) + 1, cstring1, cint, elt((ctinyint % 2) + 1, cstring1, cint) 
+FROM alltypesorc
+WHERE ctinyint > 0 LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+1	cvLH6Eat2yFsyy7p	528534767	cvLH6Eat2yFsyy7p
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+1	cvLH6Eat2yFsyy7p	528534767	cvLH6Eat2yFsyy7p
+2	cvLH6Eat2yFsyy7p	528534767	528534767
+1	cvLH6Eat2yFsyy7p	528534767	cvLH6Eat2yFsyy7p
+PREHOOK: query: EXPLAIN
+SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: alltypesorc
+            Statistics: Num rows: 0 Data size: 377237 Basic stats: PARTIAL Column stats: COMPLETE
+            Select Operator
+              expressions: elt(2, 'abc', 'defg') (type: string), elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg') (type: string), elt('1', 'abc', 'defg') (type: string), elt(2, 'aa', UDFToByte('2')) (type: string), elt(2, 'aa', UDFToShort('12345')) (type: string), elt(2, 'aa', UDFToLong('123456789012')) (type: string), elt(2, 'aa', UDFToFloat(1.25)) (type: string), elt(2, 'aa', 16.0) (type: string), elt(0, 'abc', 'defg') (type: string), elt(3, 'abc', 'defg') (type: string)
+              outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+              Statistics: Num rows: 0 Data size: 377237 Basic stats: PARTIAL Column stats: COMPLETE
+              Limit
+                Number of rows: 1
+                Statistics: Num rows: 0 Data size: 377237 Basic stats: PARTIAL Column stats: COMPLETE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 0 Data size: 377237 Basic stats: PARTIAL Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+      Execution mode: vectorized
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT elt(2, 'abc', 'defg'),
+       elt(3, 'aa', 'bb', 'cc', 'dd', 'ee', 'ff', 'gg'),
+       elt('1', 'abc', 'defg'),
+       elt(2, 'aa', CAST('2' AS TINYINT)),
+       elt(2, 'aa', CAST('12345' AS SMALLINT)),
+       elt(2, 'aa', CAST('123456789012' AS BIGINT)),
+       elt(2, 'aa', CAST(1.25 AS FLOAT)),
+       elt(2, 'aa', CAST(16.0 AS DOUBLE)),
+       elt(0, 'abc', 'defg'),
+       elt(3, 'abc', 'defg')
+FROM alltypesorc LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+defg	cc	abc	2	12345	123456789012	1.25	16.0	NULL	NULL