You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/04/15 03:11:49 UTC
svn commit: r1673599 [4/7] - in /hive/branches/cbo: ./
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hive/common/util/
hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapred...
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java Wed Apr 15 01:11:47 2015
@@ -68,10 +68,7 @@ public class VectorSelectOperator extend
* Create a new vectorization context to create a new projection, but keep
* same output column manager must be inherited to track the scratch the columns.
*/
- vOutContext = new VectorizationContext(vContext);
-
- // Set a fileKey, although this operator doesn't use it.
- vOutContext.setFileKey(vContext.getFileKey() + "/_SELECT_");
+ vOutContext = new VectorizationContext(getName(), vContext);
vOutContext.resetProjectionColumns();
for (int i=0; i < colList.size(); ++i) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Wed Apr 15 01:11:47 2015
@@ -31,8 +31,10 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.regex.Pattern;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -128,6 +130,9 @@ public class VectorizationContext {
private static final Log LOG = LogFactory.getLog(
VectorizationContext.class.getName());
+ private String contextName;
+ private int level;
+
VectorExpressionDescriptor vMap;
private List<Integer> projectedColumns;
@@ -140,7 +145,10 @@ public class VectorizationContext {
// Convenient constructor for initial batch creation takes
// a list of columns names and maps them to 0..n-1 indices.
- public VectorizationContext(List<String> initialColumnNames) {
+ public VectorizationContext(String contextName, List<String> initialColumnNames) {
+ this.contextName = contextName;
+ level = 0;
+ LOG.info("VectorizationContext consructor contextName " + contextName + " level " + level + " initialColumnNames " + initialColumnNames.toString());
this.projectionColumnNames = initialColumnNames;
projectedColumns = new ArrayList<Integer>();
@@ -157,8 +165,11 @@ public class VectorizationContext {
// Constructor to with the individual addInitialColumn method
// followed by a call to finishedAddingInitialColumns.
- public VectorizationContext() {
- projectedColumns = new ArrayList<Integer>();
+ public VectorizationContext(String contextName) {
+ this.contextName = contextName;
+ level = 0;
+ LOG.info("VectorizationContext consructor contextName " + contextName + " level " + level);
+ projectedColumns = new ArrayList<Integer>();
projectionColumnNames = new ArrayList<String>();
projectionColumnMap = new HashMap<String, Integer>();
this.ocm = new OutputColumnManager(0);
@@ -169,7 +180,10 @@ public class VectorizationContext {
// Constructor useful making a projection vectorization context.
// Use with resetProjectionColumns and addProjectionColumn.
// Keeps existing output column map, etc.
- public VectorizationContext(VectorizationContext vContext) {
+ public VectorizationContext(String contextName, VectorizationContext vContext) {
+ this.contextName = contextName;
+ level = vContext.level + 1;
+ LOG.info("VectorizationContext consructor reference contextName " + contextName + " level " + level);
this.projectedColumns = new ArrayList<Integer>();
this.projectionColumnNames = new ArrayList<String>();
this.projectionColumnMap = new HashMap<String, Integer>();
@@ -238,13 +252,6 @@ public class VectorizationContext {
//Map column number to type
private OutputColumnManager ocm;
- // File key is used by operators to retrieve the scratch vectors
- // from mapWork at runtime. The operators that modify the structure of
- // a vector row batch, need to allocate scratch vectors as well. Every
- // operator that creates a new Vectorization context should set a unique
- // fileKey.
- private String fileKey = null;
-
// Set of UDF classes for type casting data types in row-mode.
private static Set<Class<?>> castExpressionUdfs = new HashSet<Class<?>>();
static {
@@ -268,14 +275,6 @@ public class VectorizationContext {
castExpressionUdfs.add(UDFToShort.class);
}
- public String getFileKey() {
- return fileKey;
- }
-
- public void setFileKey(String fileKey) {
- this.fileKey = fileKey;
- }
-
protected int getInputColumnIndex(String name) throws HiveException {
if (name == null) {
throw new HiveException("Null column name");
@@ -316,6 +315,7 @@ public class VectorizationContext {
// We need to differentiate DECIMAL columns by their precision and scale...
String normalizedTypeName = getNormalizedName(hiveTypeName);
int relativeCol = allocateOutputColumnInternal(normalizedTypeName);
+ // LOG.info("allocateOutputColumn for hiveTypeName " + hiveTypeName + " column " + (initialOutputCol + relativeCol));
return initialOutputCol + relativeCol;
}
@@ -357,6 +357,22 @@ public class VectorizationContext {
usedOutputColumns.remove(index-initialOutputCol);
}
}
+
+ public int[] currentScratchColumns() {
+ TreeSet<Integer> treeSet = new TreeSet();
+ for (Integer col : usedOutputColumns) {
+ treeSet.add(initialOutputCol + col);
+ }
+ return ArrayUtils.toPrimitive(treeSet.toArray(new Integer[0]));
+ }
+ }
+
+ public int allocateScratchColumn(String hiveTypeName) {
+ return ocm.allocateOutputColumn(hiveTypeName);
+ }
+
+ public int[] currentScratchColumns() {
+ return ocm.currentScratchColumns();
}
private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc
@@ -2106,6 +2122,10 @@ public class VectorizationContext {
"\" for type: \"" + inputType.name() + " (reduce-side = " + isReduce + ")");
}
+ public int firstOutputColumnIndex() {
+ return firstOutputColumnIndex;
+ }
+
public Map<Integer, String> getScratchColumnTypeMap() {
Map<Integer, String> map = new HashMap<Integer, String>();
for (int i = 0; i < ocm.outputColCount; i++) {
@@ -2117,7 +2137,7 @@ public class VectorizationContext {
public String toString() {
StringBuilder sb = new StringBuilder(32);
- sb.append("Context key ").append(getFileKey()).append(", ");
+ sb.append("Context name ").append(contextName).append(", level " + level + ", ");
Comparator<Integer> comparerInteger = new Comparator<Integer>() {
@Override
@@ -2129,11 +2149,11 @@ public class VectorizationContext {
for (Map.Entry<String, Integer> entry : projectionColumnMap.entrySet()) {
sortedColumnMap.put(entry.getValue(), entry.getKey());
}
- sb.append("sortedProjectionColumnMap ").append(sortedColumnMap).append(", ");
+ sb.append("sorted projectionColumnMap ").append(sortedColumnMap).append(", ");
Map<Integer, String> sortedScratchColumnTypeMap = new TreeMap<Integer, String>(comparerInteger);
sortedScratchColumnTypeMap.putAll(getScratchColumnTypeMap());
- sb.append("sortedScratchColumnTypeMap ").append(sortedScratchColumnTypeMap);
+ sb.append("sorted scratchColumnTypeMap ").append(sortedScratchColumnTypeMap);
return sb.toString();
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java Wed Apr 15 01:11:47 2015
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.v
import java.io.IOException;
import java.sql.Timestamp;
+import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
@@ -29,7 +30,6 @@ import org.apache.hadoop.hive.common.typ
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DateWritable;
@@ -43,10 +43,15 @@ import org.apache.hadoop.hive.serde2.io.
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -542,5 +547,98 @@ public class VectorizedBatchUtil {
poi.getPrimitiveCategory());
}
}
-}
+ public static StandardStructObjectInspector convertToStandardStructObjectInspector(
+ StructObjectInspector structObjectInspector) throws HiveException {
+
+ List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs();
+ List<ObjectInspector> oids = new ArrayList<ObjectInspector>();
+ ArrayList<String> columnNames = new ArrayList<String>();
+
+ for(StructField field : fields) {
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
+ field.getFieldObjectInspector().getTypeName());
+ ObjectInspector standardWritableObjectInspector =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
+ oids.add(standardWritableObjectInspector);
+ columnNames.add(field.getFieldName());
+ }
+ return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames,oids);
+ }
+
+ public static PrimitiveTypeInfo[] primitiveTypeInfosFromStructObjectInspector(
+ StructObjectInspector structObjectInspector) throws HiveException {
+
+ List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs();
+ PrimitiveTypeInfo[] result = new PrimitiveTypeInfo[fields.size()];
+
+ int i = 0;
+ for(StructField field : fields) {
+ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
+ field.getFieldObjectInspector().getTypeName());
+ result[i++] = (PrimitiveTypeInfo) typeInfo;
+ }
+ return result;
+ }
+
+
+ public static String displayBytes(byte[] bytes, int start, int length) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = start; i < start + length; i++) {
+ char ch = (char) bytes[i];
+ if (ch < ' ' || ch > '~') {
+ sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff)));
+ } else {
+ sb.append(ch);
+ }
+ }
+ return sb.toString();
+ }
+
+ public static void debugDisplayOneRow(VectorizedRowBatch batch, int index, String prefix) {
+ StringBuffer sb = new StringBuffer();
+ sb.append(prefix + " row " + index + " ");
+ for (int i = 0; i < batch.projectionSize; i++) {
+ int column = batch.projectedColumns[i];
+ ColumnVector colVector = batch.cols[column];
+ if (colVector == null) {
+ sb.append("(null colVector " + column + ")");
+ } else {
+ boolean isRepeating = colVector.isRepeating;
+ index = (isRepeating ? 0 : index);
+ if (colVector.noNulls || !colVector.isNull[index]) {
+ if (colVector instanceof LongColumnVector) {
+ sb.append(((LongColumnVector) colVector).vector[index]);
+ } else if (colVector instanceof DoubleColumnVector) {
+ sb.append(((DoubleColumnVector) colVector).vector[index]);
+ } else if (colVector instanceof BytesColumnVector) {
+ BytesColumnVector bytesColumnVector = (BytesColumnVector) colVector;
+ byte[] bytes = bytesColumnVector.vector[index];
+ int start = bytesColumnVector.start[index];
+ int length = bytesColumnVector.length[index];
+ if (bytes == null) {
+ sb.append("(Unexpected null bytes with start " + start + " length " + length + ")");
+ } else {
+ sb.append(displayBytes(bytes, start, length));
+ }
+ } else if (colVector instanceof DecimalColumnVector) {
+ sb.append(((DecimalColumnVector) colVector).vector[index].toString());
+ } else {
+ sb.append("Unknown");
+ }
+ } else {
+ sb.append("NULL");
+ }
+ }
+ sb.append(" ");
+ }
+ System.out.println(sb.toString());
+ }
+
+ public static void debugDisplayBatch(VectorizedRowBatch batch, String prefix) throws HiveException {
+ for (int i = 0; i < batch.size; i++) {
+ int index = (batch.selectedInUse ? batch.selected[i] : i);
+ debugDisplayOneRow(batch, index, prefix);
+ }
+ }
+}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java Wed Apr 15 01:11:47 2015
@@ -190,6 +190,7 @@ public class VectorizedRowBatch implemen
* - sets size to 0
* - sets endOfFile to false
* - resets each column
+ * - inits each column
*/
public void reset() {
selectedInUse = false;
@@ -198,6 +199,7 @@ public class VectorizedRowBatch implemen
for (ColumnVector vc : cols) {
if (vc != null) {
vc.reset();
+ vc.init();
}
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Wed Apr 15 01:11:47 2015
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.exec.Ut
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.IOPrepareCache;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
@@ -174,9 +173,8 @@ public class VectorizedRowBatchCtx {
split.getPath(), IOPrepareCache.get().getPartitionDescMap());
String partitionPath = split.getPath().getParent().toString();
- scratchColumnTypeMap = Utilities
- .getMapWorkAllScratchColumnVectorTypeMaps(hiveConf)
- .get(partitionPath);
+ scratchColumnTypeMap = Utilities.getMapWorkVectorScratchColumnTypeMap(hiveConf);
+ // LOG.info("VectorizedRowBatchCtx init scratchColumnTypeMap " + scratchColumnTypeMap.toString());
Properties partProps =
(part.getPartSpec() == null || part.getPartSpec().isEmpty()) ?
@@ -631,7 +629,7 @@ public class VectorizedRowBatchCtx {
for (int i = origNumCols; i < newNumCols; i++) {
String typeName = scratchColumnTypeMap.get(i);
if (typeName == null) {
- throw new HiveException("No type found for column type entry " + i);
+ throw new HiveException("No type entry found for column " + i + " in map " + scratchColumnTypeMap.toString());
}
vrb.cols[i] = allocateColumnVector(typeName,
VectorizedRowBatch.DEFAULT_SIZE);
@@ -646,7 +644,7 @@ public class VectorizedRowBatchCtx {
* @param decimalType The given decimal type string.
* @return An integer array of size 2 with first element set to precision and second set to scale.
*/
- private int[] getScalePrecisionFromDecimalType(String decimalType) {
+ private static int[] getScalePrecisionFromDecimalType(String decimalType) {
Pattern p = Pattern.compile("\\d+");
Matcher m = p.matcher(decimalType);
m.find();
@@ -657,7 +655,7 @@ public class VectorizedRowBatchCtx {
return precScale;
}
- private ColumnVector allocateColumnVector(String type, int defaultSize) {
+ public static ColumnVector allocateColumnVector(String type, int defaultSize) {
if (type.equalsIgnoreCase("double")) {
return new DoubleColumnVector(defaultSize);
} else if (VectorizationContext.isStringFamily(type)) {
@@ -675,18 +673,4 @@ public class VectorizedRowBatchCtx {
throw new Error("Cannot allocate vector column for " + type);
}
}
-
- public VectorColumnAssign[] buildObjectAssigners(VectorizedRowBatch outputBatch)
- throws HiveException {
- List<? extends StructField> fieldRefs = rowOI.getAllStructFieldRefs();
- assert outputBatch.numCols == fieldRefs.size();
- VectorColumnAssign[] assigners = new VectorColumnAssign[fieldRefs.size()];
- for(int i = 0; i < assigners.length; ++i) {
- StructField fieldRef = fieldRefs.get(i);
- ObjectInspector fieldOI = fieldRef.getFieldObjectInspector();
- assigners[i] = VectorColumnAssignFactory.buildObjectAssign(
- outputBatch, i, fieldOI);
- }
- return assigners;
- }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Wed Apr 15 01:11:47 2015
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.DescTableDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.ShowIndexesDesc;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
/**
@@ -145,9 +147,21 @@ public final class MetaDataFormatUtils {
}
private static String convertToString(Decimal val) {
+ if (val == null) {
+ return "";
+ }
return HiveDecimal.create(new BigInteger(val.getUnscaled()), val.getScale()).toString();
}
+ private static String convertToString(org.apache.hadoop.hive.metastore.api.Date val) {
+ if (val == null) {
+ return "";
+ }
+
+ DateWritable writableValue = new DateWritable((int) val.getDaysSinceEpoch());
+ return writableValue.toString();
+ }
+
private static ColumnStatisticsObj getColumnStatisticsObject(String colName,
String colType, List<ColumnStatisticsObj> colStats) {
if (colStats != null && !colStats.isEmpty()) {
@@ -196,6 +210,12 @@ public final class MetaDataFormatUtils {
LongColumnStatsData lcsd = csd.getLongStats();
appendColumnStatsNoFormatting(colBuffer, lcsd.getLowValue(), lcsd.getHighValue(),
lcsd.getNumNulls(), lcsd.getNumDVs(), "", "", "", "");
+ } else if (csd.isSetDateStats()) {
+ DateColumnStatsData dcsd = csd.getDateStats();
+ appendColumnStatsNoFormatting(colBuffer,
+ convertToString(dcsd.getLowValue()),
+ convertToString(dcsd.getHighValue()),
+ dcsd.getNumNulls(), dcsd.getNumDVs(), "", "", "", "");
}
} else {
appendColumnStatsNoFormatting(colBuffer, "", "", "", "", "", "", "", "");
@@ -440,6 +460,12 @@ public final class MetaDataFormatUtils {
LongColumnStatsData lcsd = csd.getLongStats();
appendColumnStats(tableInfo, lcsd.getLowValue(), lcsd.getHighValue(), lcsd.getNumNulls(),
lcsd.getNumDVs(), "", "", "", "");
+ } else if (csd.isSetDateStats()) {
+ DateColumnStatsData dcsd = csd.getDateStats();
+ appendColumnStats(tableInfo,
+ convertToString(dcsd.getLowValue()),
+ convertToString(dcsd.getHighValue()),
+ dcsd.getNumNulls(), dcsd.getNumDVs(), "", "", "", "");
}
} else {
appendColumnStats(tableInfo, "", "", "", "", "", "", "", "");
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Wed Apr 15 01:11:47 2015
@@ -136,7 +136,8 @@ public class Vectorizer implements Physi
Set<String> supportedAggregationUdfs = new HashSet<String>();
- private PhysicalContext physicalContext = null;;
+ private PhysicalContext physicalContext = null;
+ private HiveConf hiveConf;
public Vectorizer() {
@@ -286,13 +287,13 @@ public class Vectorizer implements Physi
class VectorizationDispatcher implements Dispatcher {
- private final PhysicalContext pctx;
+ private final PhysicalContext physicalContext;
private List<String> reduceColumnNames;
private List<TypeInfo> reduceTypeInfos;
- public VectorizationDispatcher(PhysicalContext pctx) {
- this.pctx = pctx;
+ public VectorizationDispatcher(PhysicalContext physicalContext) {
+ this.physicalContext = physicalContext;
reduceColumnNames = null;
reduceTypeInfos = null;
}
@@ -310,7 +311,7 @@ public class Vectorizer implements Physi
convertMapWork((MapWork) w, true);
} else if (w instanceof ReduceWork) {
// We are only vectorizing Reduce under Tez.
- if (HiveConf.getBoolVar(pctx.getConf(),
+ if (HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED)) {
convertReduceWork((ReduceWork) w);
}
@@ -322,7 +323,7 @@ public class Vectorizer implements Physi
if (baseWork instanceof MapWork) {
convertMapWork((MapWork) baseWork, false);
} else if (baseWork instanceof ReduceWork
- && HiveConf.getBoolVar(pctx.getConf(),
+ && HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED)) {
convertReduceWork((ReduceWork) baseWork);
}
@@ -393,13 +394,12 @@ public class Vectorizer implements Physi
HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutput);
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps();
- mapWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps);
- Map<String, Map<String, Integer>> allColumnVectorMaps = vnp.getAllColumnVectorMaps();
- mapWork.setAllColumnVectorMaps(allColumnVectorMaps);
+ mapWork.setVectorColumnNameMap(vnp.getVectorColumnNameMap());
+ mapWork.setVectorColumnTypeMap(vnp.getVectorColumnTypeMap());
+ mapWork.setVectorScratchColumnTypeMap(vnp.getVectorScratchColumnTypeMap());
if (LOG.isDebugEnabled()) {
- debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps);
+ debugDisplayAllMaps(mapWork);
}
return;
@@ -495,7 +495,7 @@ public class Vectorizer implements Physi
// VectorizationContext... Do we use PreOrderWalker instead of DefaultGraphWalker.
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
ReduceWorkVectorizationNodeProcessor vnp =
- new ReduceWorkVectorizationNodeProcessor(reduceColumnNames);
+ new ReduceWorkVectorizationNodeProcessor(reduceColumnNames, reduceTypeInfos);
addReduceWorkRules(opRules, vnp);
Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
GraphWalker ogw = new PreOrderWalker(disp);
@@ -510,14 +510,12 @@ public class Vectorizer implements Physi
// Necessary since we are vectorizing the root operator in reduce.
reduceWork.setReducer(vnp.getRootVectorOp());
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps();
- reduceWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps);
- Map<String, Map<String, Integer>> allColumnVectorMaps = vnp.getAllColumnVectorMaps();
- reduceWork.setAllColumnVectorMaps(allColumnVectorMaps);
-
+ reduceWork.setVectorColumnNameMap(vnp.getVectorColumnNameMap());
+ reduceWork.setVectorColumnTypeMap(vnp.getVectorColumnTypeMap());
+ reduceWork.setVectorScratchColumnTypeMap(vnp.getVectorScratchColumnTypeMap());
if (LOG.isDebugEnabled()) {
- debugDisplayAllMaps(allColumnVectorMaps, allScratchColumnVectorTypeMaps);
+ debugDisplayAllMaps(reduceWork);
}
}
}
@@ -574,37 +572,33 @@ public class Vectorizer implements Physi
// ReduceWorkVectorizationNodeProcessor.
class VectorizationNodeProcessor implements NodeProcessor {
- // This is used to extract scratch column types for each file key
- protected final Map<String, VectorizationContext> scratchColumnContext =
- new HashMap<String, VectorizationContext>();
+ // The vectorization context for the Map or Reduce task.
+ protected VectorizationContext taskVectorizationContext;
+
+ // The input projection column type name map for the Map or Reduce task.
+ protected Map<Integer, String> taskColumnTypeNameMap;
+
+ VectorizationNodeProcessor() {
+ taskColumnTypeNameMap = new HashMap<Integer, String>();
+ }
- protected final Map<Operator<? extends OperatorDesc>, VectorizationContext> vContextsByOp =
- new HashMap<Operator<? extends OperatorDesc>, VectorizationContext>();
+ public Map<String, Integer> getVectorColumnNameMap() {
+ return taskVectorizationContext.getProjectionColumnMap();
+ }
+
+ public Map<Integer, String> getVectorColumnTypeMap() {
+ return taskColumnTypeNameMap;
+ }
+
+ public Map<Integer, String> getVectorScratchColumnTypeMap() {
+ return taskVectorizationContext.getScratchColumnTypeMap();
+ }
protected final Set<Operator<? extends OperatorDesc>> opsDone =
new HashSet<Operator<? extends OperatorDesc>>();
- public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps =
- new HashMap<String, Map<Integer, String>>();
- for (String onefile : scratchColumnContext.keySet()) {
- VectorizationContext vc = scratchColumnContext.get(onefile);
- Map<Integer, String> cmap = vc.getScratchColumnTypeMap();
- allScratchColumnVectorTypeMaps.put(onefile, cmap);
- }
- return allScratchColumnVectorTypeMaps;
- }
-
- public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
- Map<String, Map<String, Integer>> allColumnVectorMaps =
- new HashMap<String, Map<String, Integer>>();
- for(String oneFile: scratchColumnContext.keySet()) {
- VectorizationContext vc = scratchColumnContext.get(oneFile);
- Map<String, Integer> cmap = vc.getProjectionColumnMap();
- allColumnVectorMaps.put(oneFile, cmap);
- }
- return allColumnVectorMaps;
- }
+ protected final Map<Operator<? extends OperatorDesc>, Operator<? extends OperatorDesc>> opToVectorOpMap =
+ new HashMap<Operator<? extends OperatorDesc>, Operator<? extends OperatorDesc>>();
public VectorizationContext walkStackToFindVectorizationContext(Stack<Node> stack,
Operator<? extends OperatorDesc> op) throws SemanticException {
@@ -622,7 +616,18 @@ public class Vectorizer implements Physi
return null;
}
Operator<? extends OperatorDesc> opParent = (Operator<? extends OperatorDesc>) stack.get(i);
- vContext = vContextsByOp.get(opParent);
+ Operator<? extends OperatorDesc> vectorOpParent = opToVectorOpMap.get(opParent);
+ if (vectorOpParent != null) {
+ if (vectorOpParent instanceof VectorizationContextRegion) {
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOpParent;
+ vContext = vcRegion.getOuputVectorizationContext();
+ LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " has new vectorization context " + vContext.toString());
+ } else {
+ LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " does not have new vectorization context");
+ }
+ } else {
+ LOG.info("walkStackToFindVectorizationContext " + opParent.getName() + " is not vectorized");
+ }
--i;
}
return vContext;
@@ -636,14 +641,9 @@ public class Vectorizer implements Physi
vectorOp = vectorizeOperator(op, vContext);
opsDone.add(op);
if (vectorOp != op) {
+ opToVectorOpMap.put(op, vectorOp);
opsDone.add(vectorOp);
}
- if (vectorOp instanceof VectorizationContextRegion) {
- VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
- VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
- vContextsByOp.put(op, vOutContext);
- scratchColumnContext.put(vOutContext.getFileKey(), vOutContext);
- }
}
} catch (HiveException e) {
throw new SemanticException(e);
@@ -663,6 +663,7 @@ public class Vectorizer implements Physi
private final MapWork mWork;
public MapWorkVectorizationNodeProcessor(MapWork mWork) {
+ super();
this.mWork = mWork;
}
@@ -671,41 +672,26 @@ public class Vectorizer implements Physi
Object... nodeOutputs) throws SemanticException {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
- LOG.info("MapWorkVectorizationNodeProcessor processing Operator: " + op.getName() + "...");
VectorizationContext vContext = null;
if (op instanceof TableScanOperator) {
- vContext = getVectorizationContext(op, physicalContext);
- for (String onefile : mWork.getPathToAliases().keySet()) {
- List<String> aliases = mWork.getPathToAliases().get(onefile);
- for (String alias : aliases) {
- Operator<? extends OperatorDesc> opRoot = mWork.getAliasToWork().get(alias);
- if (op == opRoot) {
- // The same vectorization context is copied multiple times into
- // the MapWork scratch columnMap
- // Each partition gets a copy
- //
- vContext.setFileKey(onefile);
- scratchColumnContext.put(onefile, vContext);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Vectorized MapWork operator " + op.getName() + " vectorization context " + vContext.toString());
- }
- break;
- }
- }
+ if (taskVectorizationContext == null) {
+ taskVectorizationContext = getVectorizationContext(op.getSchema(), op.getName(),
+ taskColumnTypeNameMap);
}
- vContextsByOp.put(op, vContext);
+ vContext = taskVectorizationContext;
} else {
+ LOG.info("MapWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName());
vContext = walkStackToFindVectorizationContext(stack, op);
if (vContext == null) {
- throw new SemanticException(
- String.format("Did not find vectorization context for operator %s in operator stack",
- op.getName()));
+ // No operator has "pushed" a new context -- so use the task vectorization context.
+ vContext = taskVectorizationContext;
}
}
assert vContext != null;
+ LOG.info("MapWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString());
// When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
// vectorize the operators below it.
@@ -720,9 +706,10 @@ public class Vectorizer implements Physi
Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
if (LOG.isDebugEnabled()) {
- LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
if (vectorOp instanceof VectorizationContextRegion) {
- LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+ VectorizationContext vNewContext = vcRegion.getOuputVectorizationContext();
+ LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vNewContext.toString());
}
}
@@ -733,8 +720,7 @@ public class Vectorizer implements Physi
class ReduceWorkVectorizationNodeProcessor extends VectorizationNodeProcessor {
private final List<String> reduceColumnNames;
-
- private VectorizationContext reduceShuffleVectorizationContext;
+ private final List<TypeInfo> reduceTypeInfos;
private Operator<? extends OperatorDesc> rootVectorOp;
@@ -742,10 +728,12 @@ public class Vectorizer implements Physi
return rootVectorOp;
}
- public ReduceWorkVectorizationNodeProcessor(List<String> reduceColumnNames) {
+ public ReduceWorkVectorizationNodeProcessor(List<String> reduceColumnNames,
+ List<TypeInfo> reduceTypeInfos) {
+ super();
this.reduceColumnNames = reduceColumnNames;
+ this.reduceTypeInfos = reduceTypeInfos;
rootVectorOp = null;
- reduceShuffleVectorizationContext = null;
}
@Override
@@ -753,8 +741,6 @@ public class Vectorizer implements Physi
Object... nodeOutputs) throws SemanticException {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
- LOG.info("ReduceWorkVectorizationNodeProcessor processing Operator: " +
- op.getName() + "...");
VectorizationContext vContext = null;
@@ -763,25 +749,30 @@ public class Vectorizer implements Physi
if (op.getParentOperators().size() == 0) {
LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + reduceColumnNames.toString());
- vContext = new VectorizationContext(reduceColumnNames);
- vContext.setFileKey("_REDUCE_SHUFFLE_");
- scratchColumnContext.put("_REDUCE_SHUFFLE_", vContext);
- reduceShuffleVectorizationContext = vContext;
+ vContext = new VectorizationContext("__Reduce_Shuffle__", reduceColumnNames);
+ taskVectorizationContext = vContext;
+ int i = 0;
+ for (TypeInfo typeInfo : reduceTypeInfos) {
+ taskColumnTypeNameMap.put(i, typeInfo.getTypeName());
+ i++;
+ }
saveRootVectorOp = true;
if (LOG.isDebugEnabled()) {
LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context " + vContext.toString());
}
} else {
+ LOG.info("ReduceWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName());
vContext = walkStackToFindVectorizationContext(stack, op);
if (vContext == null) {
// If we didn't find a context among the operators, assume the top -- reduce shuffle's
// vectorization context.
- vContext = reduceShuffleVectorizationContext;
+ vContext = taskVectorizationContext;
}
}
assert vContext != null;
+ LOG.info("ReduceWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString());
// When Vectorized GROUPBY outputs rows instead of vectorized row batchs, we don't
// vectorize the operators below it.
@@ -796,9 +787,10 @@ public class Vectorizer implements Physi
Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
if (LOG.isDebugEnabled()) {
- LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString());
if (vectorOp instanceof VectorizationContextRegion) {
- LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString());
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+ VectorizationContext vNewContext = vcRegion.getOuputVectorizationContext();
+ LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vNewContext.toString());
}
}
if (vectorOp instanceof VectorGroupByOperator) {
@@ -816,7 +808,7 @@ public class Vectorizer implements Physi
private static class ValidatorVectorizationContext extends VectorizationContext {
private ValidatorVectorizationContext() {
- super();
+ super("No Name");
}
@Override
@@ -831,25 +823,27 @@ public class Vectorizer implements Physi
}
@Override
- public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException {
- this.physicalContext = pctx;
- boolean vectorPath = HiveConf.getBoolVar(pctx.getConf(),
+ public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
+ this.physicalContext = physicalContext;
+ hiveConf = physicalContext.getConf();
+
+ boolean vectorPath = HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
if (!vectorPath) {
LOG.info("Vectorization is disabled");
- return pctx;
+ return physicalContext;
}
// create dispatcher and graph walker
- Dispatcher disp = new VectorizationDispatcher(pctx);
+ Dispatcher disp = new VectorizationDispatcher(physicalContext);
TaskGraphWalker ogw = new TaskGraphWalker(disp);
// get all the tasks nodes from root task
ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(pctx.getRootTasks());
+ topNodes.addAll(physicalContext.getRootTasks());
// begin to walk through the task tree.
ogw.startWalking(topNodes, null);
- return pctx;
+ return physicalContext;
}
boolean validateMapWorkOperator(Operator<? extends OperatorDesc> op, MapWork mWork, boolean isTez) {
@@ -901,7 +895,7 @@ public class Vectorizer implements Physi
}
break;
case GROUPBY:
- if (HiveConf.getBoolVar(physicalContext.getConf(),
+ if (HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED)) {
ret = validateGroupByOperator((GroupByOperator) op, true, true);
} else {
@@ -1262,20 +1256,24 @@ public class Vectorizer implements Physi
return supportedDataTypesPattern.matcher(type.toLowerCase()).matches();
}
- private VectorizationContext getVectorizationContext(Operator op,
- PhysicalContext pctx) {
- RowSchema rs = op.getSchema();
+ private VectorizationContext getVectorizationContext(RowSchema rowSchema, String contextName,
+ Map<Integer, String> typeNameMap) {
+
+ VectorizationContext vContext = new VectorizationContext(contextName);
// Add all non-virtual columns to make a vectorization context for
// the TableScan operator.
- VectorizationContext vContext = new VectorizationContext();
- for (ColumnInfo c : rs.getSignature()) {
+ int i = 0;
+ for (ColumnInfo c : rowSchema.getSignature()) {
// Earlier, validation code should have eliminated virtual columns usage (HIVE-5560).
if (!isVirtualColumn(c)) {
vContext.addInitialColumn(c.getInternalName());
+ typeNameMap.put(i, c.getTypeName());
+ i++;
}
}
vContext.finishedAddingInitialColumns();
+
return vContext;
}
@@ -1333,40 +1331,14 @@ public class Vectorizer implements Physi
return false;
}
- public void debugDisplayAllMaps(Map<String, Map<String, Integer>> allColumnVectorMaps,
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps) {
+ public void debugDisplayAllMaps(BaseWork work) {
- // Context keys grow in length since they are a path...
- Comparator<String> comparerShorterString = new Comparator<String>() {
- @Override
- public int compare(String o1, String o2) {
- Integer length1 = o1.length();
- Integer length2 = o2.length();
- return length1.compareTo(length2);
- }};
-
- Comparator<Integer> comparerInteger = new Comparator<Integer>() {
- @Override
- public int compare(Integer o1, Integer o2) {
- return o1.compareTo(o2);
- }};
-
- Map<String, Map<Integer, String>> sortedAllColumnVectorMaps = new TreeMap<String, Map<Integer, String>>(comparerShorterString);
- for (Map.Entry<String, Map<String, Integer>> entry : allColumnVectorMaps.entrySet()) {
- Map<Integer, String> sortedColumnMap = new TreeMap<Integer, String>(comparerInteger);
- for (Map.Entry<String, Integer> innerEntry : entry.getValue().entrySet()) {
- sortedColumnMap.put(innerEntry.getValue(), innerEntry.getKey());
- }
- sortedAllColumnVectorMaps.put(entry.getKey(), sortedColumnMap);
- }
- LOG.debug("sortedAllColumnVectorMaps " + sortedAllColumnVectorMaps);
-
- Map<String, Map<Integer, String>> sortedAllScratchColumnVectorTypeMap = new TreeMap<String, Map<Integer, String>>(comparerShorterString);
- for (Map.Entry<String, Map<Integer, String>> entry : allScratchColumnVectorTypeMaps.entrySet()) {
- Map<Integer, String> sortedScratchColumnTypeMap = new TreeMap<Integer, String>(comparerInteger);
- sortedScratchColumnTypeMap.putAll(entry.getValue());
- sortedAllScratchColumnVectorTypeMap.put(entry.getKey(), sortedScratchColumnTypeMap);
- }
- LOG.debug("sortedAllScratchColumnVectorTypeMap " + sortedAllScratchColumnVectorTypeMap);
+ Map<String, Integer> columnNameMap = work.getVectorColumnNameMap();
+ Map<Integer, String> columnTypeMap = work.getVectorColumnTypeMap();
+ Map<Integer, String> scratchColumnTypeMap = work.getVectorScratchColumnTypeMap();
+
+ LOG.debug("debugDisplayAllMaps columnNameMap " + columnNameMap.toString());
+ LOG.debug("debugDisplayAllMaps columnTypeMap " + columnTypeMap.toString());
+ LOG.debug("debugDisplayAllMaps scratchColumnTypeMap " + scratchColumnTypeMap.toString());
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Wed Apr 15 01:11:47 2015
@@ -196,18 +196,13 @@ public class PartitionPruner implements
// Remove all parts that are not partition columns. See javadoc for details.
ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
String oldFilter = prunerExpr.getExprString();
- if (isBooleanExpr(compactExpr)) {
- // For null and true values, return every partition
- if (!isFalseExpr(compactExpr)) {
- // Non-strict mode, and all the predicates are on non-partition columns - get everything.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Filter " + oldFilter + " was null after compacting");
- }
- return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
- } else {
- return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(new ArrayList<Partition>()),
- new ArrayList<String>(), false);
- }
+ if (compactExpr == null || isBooleanExpr(compactExpr)) {
+ if (isFalseExpr(compactExpr)) {
+ return new PrunedPartitionList(
+ tab, new LinkedHashSet<Partition>(0), new ArrayList<String>(0), false);
+ }
+ // For null and true values, return every partition
+ return getAllPartsFromCacheOrServer(tab, key, true, prunedPartitionsMap);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
@@ -241,22 +236,22 @@ public class PartitionPruner implements
partsCache.put(key, ppList);
return ppList;
}
-
+
static private boolean isBooleanExpr(ExprNodeDesc expr) {
- return expr != null && expr instanceof ExprNodeConstantDesc &&
+ return expr != null && expr instanceof ExprNodeConstantDesc &&
((ExprNodeConstantDesc)expr).getTypeInfo() instanceof PrimitiveTypeInfo &&
((PrimitiveTypeInfo)(((ExprNodeConstantDesc)expr).getTypeInfo())).
- getTypeName().equals(serdeConstants.BOOLEAN_TYPE_NAME);
+ getTypeName().equals(serdeConstants.BOOLEAN_TYPE_NAME);
}
static private boolean isTrueExpr(ExprNodeDesc expr) {
- return isBooleanExpr(expr) &&
- ((ExprNodeConstantDesc)expr).getValue() != null &&
- ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.TRUE);
+ return isBooleanExpr(expr) &&
+ ((ExprNodeConstantDesc)expr).getValue() != null &&
+ ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.TRUE);
}
static private boolean isFalseExpr(ExprNodeDesc expr) {
- return isBooleanExpr(expr) &&
+ return isBooleanExpr(expr) &&
((ExprNodeConstantDesc)expr).getValue() != null &&
- ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.FALSE);
+ ((ExprNodeConstantDesc)expr).getValue().equals(Boolean.FALSE);
}
/**
@@ -268,42 +263,48 @@ public class PartitionPruner implements
*/
static private ExprNodeDesc compactExpr(ExprNodeDesc expr) {
// If this is a constant boolean expression, return the value.
- if (expr == null) {
- return null;
- }
- if (expr instanceof ExprNodeConstantDesc) {
- if (isBooleanExpr(expr)) {
- return expr;
- } else {
- throw new IllegalStateException("Unexpected non-null ExprNodeConstantDesc: "
- + expr.getExprString());
+ if (expr == null) {
+ return null;
+ }
+ if (expr instanceof ExprNodeConstantDesc) {
+ if (((ExprNodeConstantDesc)expr).getValue() == null) return null;
+ if (!isBooleanExpr(expr)) {
+ throw new IllegalStateException("Unexpected non-boolean ExprNodeConstantDesc: "
+ + expr.getExprString());
}
+ return expr;
} else if (expr instanceof ExprNodeGenericFuncDesc) {
GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
boolean isAnd = udf instanceof GenericUDFOPAnd;
boolean isOr = udf instanceof GenericUDFOPOr;
-
+
if (isAnd || isOr) {
List<ExprNodeDesc> children = expr.getChildren();
- ExprNodeDesc left = children.get(0);
- children.set(0, compactExpr(left));
- ExprNodeDesc right = children.get(1);
- children.set(1, compactExpr(right));
-
- if (isTrueExpr(children.get(0)) && isTrueExpr(children.get(1))) {
- return new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isTrueExpr(children.get(0))) {
- return isAnd ? children.get(1) : new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isTrueExpr(children.get(1))) {
- return isAnd ? children.get(0) : new ExprNodeConstantDesc(Boolean.TRUE);
- } else if (isFalseExpr(children.get(0)) && isFalseExpr(children.get(1))) {
- return new ExprNodeConstantDesc(Boolean.FALSE);
- } else if (isFalseExpr(children.get(0))) {
- return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(1);
- } else if (isFalseExpr(children.get(1))) {
- return isAnd ? new ExprNodeConstantDesc(Boolean.FALSE) : children.get(0);
- }
-
+ ExprNodeDesc left = compactExpr(children.get(0));
+ ExprNodeDesc right = compactExpr(children.get(1));
+ // Non-partition expressions are converted to nulls.
+ if (left == null && right == null) {
+ return null;
+ } else if (left == null) {
+ return isAnd ? right : null;
+ } else if (right == null) {
+ return isAnd ? left : null;
+ }
+ // Handle boolean expressions
+ boolean isLeftFalse = isFalseExpr(left), isRightFalse = isFalseExpr(right),
+ isLeftTrue = isTrueExpr(left), isRightTrue = isTrueExpr(right);
+ if ((isRightTrue && isLeftTrue) || (isOr && (isLeftTrue || isRightTrue))) {
+ return new ExprNodeConstantDesc(Boolean.TRUE);
+ } else if ((isRightFalse && isLeftFalse) || (isAnd && (isLeftFalse || isRightFalse))) {
+ return new ExprNodeConstantDesc(Boolean.FALSE);
+ } else if ((isAnd && isLeftTrue) || (isOr && isLeftFalse)) {
+ return right;
+ } else if ((isAnd && isRightTrue) || (isOr && isRightFalse)) {
+ return left;
+ }
+ // Nothing to compact, update expr with compacted children.
+ children.set(0, left);
+ children.set(1, right);
}
return expr;
} else {
@@ -328,9 +329,9 @@ public class PartitionPruner implements
if (!partCols.contains(column)) {
// Column doesn't appear to be a partition column for the table.
return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
- }
+ }
referred.add(column);
- }
+ }
if (expr instanceof ExprNodeGenericFuncDesc) {
List<ExprNodeDesc> children = expr.getChildren();
for (int i = 0; i < children.size(); ++i) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Wed Apr 15 01:11:47 2015
@@ -60,8 +60,9 @@ public abstract class BaseWork extends A
private String name;
// Vectorization.
- protected Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = null;
- protected Map<String, Map<String, Integer>> allColumnVectorMaps = null;
+ protected Map<String, Integer> vectorColumnNameMap;
+ protected Map<Integer, String> vectorColumnTypeMap;
+ protected Map<Integer, String> vectorScratchColumnTypeMap;
public void setGatheringStats(boolean gatherStats) {
this.gatheringStats = gatherStats;
@@ -143,21 +144,28 @@ public abstract class BaseWork extends A
return returnSet;
}
- public Map<String, Map<Integer, String>> getAllScratchColumnVectorTypeMaps() {
- return allScratchColumnVectorTypeMaps;
+ public Map<String, Integer> getVectorColumnNameMap() {
+ return vectorColumnNameMap;
}
- public void setAllScratchColumnVectorTypeMaps(
- Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps) {
- this.allScratchColumnVectorTypeMaps = allScratchColumnVectorTypeMaps;
+ public void setVectorColumnNameMap(Map<String, Integer> vectorColumnNameMap) {
+ this.vectorColumnNameMap = vectorColumnNameMap;
}
- public Map<String, Map<String, Integer>> getAllColumnVectorMaps() {
- return allColumnVectorMaps;
+ public Map<Integer, String> getVectorColumnTypeMap() {
+ return vectorColumnTypeMap;
}
- public void setAllColumnVectorMaps(Map<String, Map<String, Integer>> allColumnVectorMaps) {
- this.allColumnVectorMaps = allColumnVectorMaps;
+ public void setVectorColumnTypeMap(Map<Integer, String> vectorColumnTypeMap) {
+ this.vectorColumnTypeMap = vectorColumnTypeMap;
+ }
+
+ public Map<Integer, String> getVectorScratchColumnTypeMap() {
+ return vectorScratchColumnTypeMap;
+ }
+
+ public void setVectorScratchColumnTypeMap(Map<Integer, String> vectorScratchColumnTypeMap) {
+ this.vectorScratchColumnTypeMap = vectorScratchColumnTypeMap;
}
/**
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java Wed Apr 15 01:11:47 2015
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -86,9 +87,11 @@ public class GenericUDAFComputeStats ext
return new GenericUDAFBinaryStatsEvaluator();
case DECIMAL:
return new GenericUDAFDecimalStatsEvaluator();
+ case DATE:
+ return new GenericUDAFDateStatsEvaluator();
default:
throw new UDFArgumentTypeException(0,
- "Only integer/long/timestamp/float/double/string/binary/boolean/decimal type argument " +
+ "Only integer/long/timestamp/date/float/double/string/binary/boolean/decimal type argument " +
"is accepted but "
+ parameters[0].getTypeName() + " is passed.");
}
@@ -1314,4 +1317,73 @@ public class GenericUDAFComputeStats ext
((NumericStatsAgg)agg).reset("Decimal");
}
}
+
+ /**
+ * GenericUDAFDateStatsEvaluator
+ * High/low value will be saved in stats DB as long value representing days since epoch.
+ */
+ public static class GenericUDAFDateStatsEvaluator
+ extends GenericUDAFNumericStatsEvaluator<DateWritable, DateObjectInspector> {
+
+ @Override
+ protected DateObjectInspector getValueObjectInspector() {
+ return PrimitiveObjectInspectorFactory.writableDateObjectInspector;
+ }
+
+ @AggregationType(estimable = true)
+ public class DateStatsAgg extends NumericStatsAgg {
+ @Override
+ public int estimate() {
+ JavaDataModel model = JavaDataModel.get();
+ return super.estimate() + model.primitive2() * 2;
+ }
+
+ @Override
+ protected void update(Object p, PrimitiveObjectInspector inputOI) {
+ // DateWritable is mutable, DateStatsAgg needs its own copy
+ DateWritable v = new DateWritable((DateWritable) inputOI.getPrimitiveWritableObject(p));
+
+ //Update min counter if new value is less than min seen so far
+ if (min == null || v.compareTo(min) < 0) {
+ min = v;
+ }
+ //Update max counter if new value is greater than max seen so far
+ if (max == null || v.compareTo(max) > 0) {
+ max = v;
+ }
+ // Add value to NumDistinctValue Estimator
+ numDV.addToEstimator(v.getDays());
+ }
+
+ @Override
+ protected void updateMin(Object minValue, DateObjectInspector minFieldOI) {
+ if ((minValue != null) && (min == null ||
+ min.compareTo(minFieldOI.getPrimitiveWritableObject(minValue)) > 0)) {
+ // DateWritable is mutable, DateStatsAgg needs its own copy
+ min = new DateWritable(minFieldOI.getPrimitiveWritableObject(minValue));
+ }
+ }
+
+ @Override
+ protected void updateMax(Object maxValue, DateObjectInspector maxFieldOI) {
+ if ((maxValue != null) && (max == null ||
+ max.compareTo(maxFieldOI.getPrimitiveWritableObject(maxValue)) < 0)) {
+ // DateWritable is mutable, DateStatsAgg needs its own copy
+ max = new DateWritable(maxFieldOI.getPrimitiveWritableObject(maxValue));
+ }
+ }
+ };
+
+ @Override
+ public AggregationBuffer getNewAggregationBuffer() throws HiveException {
+ AggregationBuffer result = new DateStatsAgg();
+ reset(result);
+ return result;
+ }
+
+ @Override
+ public void reset(AggregationBuffer agg) throws HiveException {
+ ((NumericStatsAgg)agg).reset("Date");
+ }
+ }
}
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java Wed Apr 15 01:11:47 2015
@@ -88,7 +88,7 @@ public class TestVectorFilterOperator {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false);
List<String> columns = new ArrayList<String>();
columns.add("col1");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
FilterDesc fdesc = new FilterDesc();
fdesc.setPredicate(col1Expr);
return new VectorFilterOperator(vc, fdesc);
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java Wed Apr 15 01:11:47 2015
@@ -173,7 +173,7 @@ public class TestVectorGroupByOperator {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("Key");
mapColumnNames.add("Value");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildKeyGroupByDesc (ctx, "max",
"Value", TypeInfoFactory.longTypeInfo,
@@ -1710,7 +1710,7 @@ public class TestVectorGroupByOperator {
mapColumnNames.put("value", i);
outputColumnNames.add("value");
- VectorizationContext ctx = new VectorizationContext(outputColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", outputColumnNames);
ArrayList<AggregationDesc> aggs = new ArrayList(1);
aggs.add(
@@ -1821,7 +1821,7 @@ public class TestVectorGroupByOperator {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("Key");
mapColumnNames.add("Value");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
Set<Object> keys = new HashSet<Object>();
AggregationDesc agg = buildAggregationDesc(ctx, aggregateName,
@@ -2235,7 +2235,7 @@ public class TestVectorGroupByOperator {
Object expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescCountStar (ctx);
@@ -2264,7 +2264,7 @@ public class TestVectorGroupByOperator {
Object expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescType(ctx, "count", "A", TypeInfoFactory.longTypeInfo);
VectorGroupByDesc vectorDesc = desc.getVectorDesc();
@@ -2296,7 +2296,7 @@ public class TestVectorGroupByOperator {
Object expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A",
TypeInfoFactory.stringTypeInfo);
@@ -2322,11 +2322,12 @@ public class TestVectorGroupByOperator {
}
public void testAggregateDecimalIterable (
-String aggregateName, Iterable<VectorizedRowBatch> data,
- Object expected) throws HiveException {
- List<String> mapColumnNames = new ArrayList<String>();
- mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ String aggregateName,
+ Iterable<VectorizedRowBatch> data,
+ Object expected) throws HiveException {
+ List<String> mapColumnNames = new ArrayList<String>();
+ mapColumnNames.add("A");
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc =
buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4));
@@ -2358,7 +2359,7 @@ String aggregateName, Iterable<Vectorize
Object expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescType (ctx, aggregateName, "A",
TypeInfoFactory.doubleTypeInfo);
@@ -2389,7 +2390,7 @@ String aggregateName, Iterable<Vectorize
Object expected) throws HiveException {
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.longTypeInfo);
@@ -2420,7 +2421,7 @@ String aggregateName, Iterable<Vectorize
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("Key");
mapColumnNames.add("Value");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
Set<Object> keys = new HashSet<Object>();
@@ -2487,7 +2488,7 @@ String aggregateName, Iterable<Vectorize
List<String> mapColumnNames = new ArrayList<String>();
mapColumnNames.add("Key");
mapColumnNames.add("Value");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
+ VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
Set<Object> keys = new HashSet<Object>();
GroupByDesc desc = buildKeyGroupByDesc (ctx, aggregateName, "Value",
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java Wed Apr 15 01:11:47 2015
@@ -88,7 +88,7 @@ public class TestVectorSelectOperator {
columns.add("a");
columns.add("b");
columns.add("c");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
SelectDesc selDesc = new SelectDesc(false);
List<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Wed Apr 15 01:11:47 2015
@@ -247,7 +247,7 @@ public class TestVectorizationContext {
children5.add(col6Expr);
modExpr.setChildren(children5);
- VectorizationContext vc = new VectorizationContext();
+ VectorizationContext vc = new VectorizationContext("name");
vc.addInitialColumn("col1");
vc.addInitialColumn("col2");
vc.addInitialColumn("col3");
@@ -297,7 +297,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -322,7 +322,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -341,7 +341,7 @@ public class TestVectorizationContext {
children1.add(col2Expr);
exprDesc.setChildren(children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -360,7 +360,7 @@ public class TestVectorizationContext {
children1.add(col2Expr);
exprDesc.setChildren(children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -378,7 +378,7 @@ public class TestVectorizationContext {
children1.add(col2Expr);
exprDesc.setChildren(children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -395,7 +395,7 @@ public class TestVectorizationContext {
children1.add(col2Expr);
exprDesc.setChildren(children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -412,7 +412,7 @@ public class TestVectorizationContext {
children1.add(col2Expr);
exprDesc.setChildren(children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -434,7 +434,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col1");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
@@ -480,7 +480,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -530,7 +530,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression veAnd = vc.getVectorExpression(andExprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veAnd.getClass(), FilterExprAndExpr.class);
assertEquals(veAnd.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
@@ -555,7 +555,7 @@ public class TestVectorizationContext {
orExprDesc.setChildren(children4);
//Allocate new Vectorization context to reset the intermediate columns.
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
VectorExpression veOr = vc.getVectorExpression(orExprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(veOr.getClass(), FilterExprOrExpr.class);
assertEquals(veOr.getChildExpressions()[0].getClass(), FilterLongColGreaterLongScalar.class);
@@ -596,7 +596,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(notExpr, VectorExpressionDescriptor.Mode.FILTER);
@@ -633,7 +633,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(isNullExpr, VectorExpressionDescriptor.Mode.FILTER);
@@ -674,7 +674,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(isNotNullExpr, VectorExpressionDescriptor.Mode.FILTER);
@@ -703,7 +703,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(scalarMinusConstant, VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(ve.getClass(), LongScalarSubtractLongColumn.class);
@@ -726,7 +726,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
@@ -744,7 +744,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col0");
columns.add("col1");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
@@ -762,7 +762,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("col0");
columns.add("col1");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(negExprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
@@ -787,7 +787,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(scalarGreaterColExpr, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterLongScalarGreaterLongColumn.class, ve.getClass());
}
@@ -810,7 +810,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.FILTER);
assertEquals(FilterLongColEqualLongScalar.class, ve.getClass());
}
@@ -833,7 +833,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(colEqualScalar, VectorExpressionDescriptor.Mode.PROJECTION);
assertEquals(LongColEqualLongScalar.class, ve.getClass());
}
@@ -850,7 +850,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("b");
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
GenericUDF stringLower = new GenericUDFLower();
stringUnary.setGenericUDF(stringLower);
@@ -860,7 +860,7 @@ public class TestVectorizationContext {
assertEquals(1, ((StringLower) ve).getColNum());
assertEquals(2, ((StringLower) ve).getOutputColumn());
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ExprNodeGenericFuncDesc anotherUnary = new ExprNodeGenericFuncDesc();
anotherUnary.setTypeInfo(TypeInfoFactory.stringTypeInfo);
@@ -895,7 +895,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("b");
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
// Sin(double)
GenericUDFBridge gudfBridge = new GenericUDFBridge("sin", false, UDFSin.class.getName());
@@ -986,7 +986,7 @@ public class TestVectorizationContext {
List<String> columns = new ArrayList<String>();
columns.add("b");
columns.add("a");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
//UDFYear
GenericUDFBridge gudfBridge = new GenericUDFBridge("year", false, UDFYear.class.getName());
@@ -1024,7 +1024,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterStringColumnBetween);
@@ -1050,7 +1050,7 @@ public class TestVectorizationContext {
exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf,
children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterCharColumnBetween);
@@ -1075,7 +1075,7 @@ public class TestVectorizationContext {
exprDesc = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, udf,
children1);
- vc = new VectorizationContext(columns);
+ vc = new VectorizationContext("name", columns);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterVarCharColumnBetween);
@@ -1144,7 +1144,7 @@ public class TestVectorizationContext {
columns.add("col0");
columns.add("col1");
columns.add("col2");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.FILTER);
assertTrue(ve instanceof FilterStringColumnInList);
ve = vc.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
@@ -1199,7 +1199,7 @@ public class TestVectorizationContext {
columns.add("col1");
columns.add("col2");
columns.add("col3");
- VectorizationContext vc = new VectorizationContext(columns);
+ VectorizationContext vc = new VectorizationContext("name", columns);
VectorExpression ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprLongColumnLongColumn);
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Wed Apr 15 01:11:47 2015
@@ -1288,9 +1288,6 @@ public class TestInputOutputFormat {
}
mapWork.setPathToAliases(aliasMap);
mapWork.setPathToPartitionInfo(partMap);
- mapWork.setAllColumnVectorMaps(new HashMap<String, Map<String, Integer>>());
- mapWork.setAllScratchColumnVectorTypeMaps(new HashMap<String,
- Map<Integer, String>>());
// write the plan out
FileSystem localFs = FileSystem.getLocal(conf).getRaw();
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Wed Apr 15 01:11:47 2015
@@ -187,10 +187,11 @@ public class TestOrcFile {
}
private static ByteBuffer byteBuf(int... items) {
- ByteBuffer result = ByteBuffer.allocate(items.length);
+ ByteBuffer result = ByteBuffer.allocate(items.length);
for(int item: items) {
result.put((byte) item);
}
+ result.flip();
return result;
}
@@ -703,12 +704,12 @@ public class TestOrcFile {
assertEquals(0, items.get(0).getPositions(0));
assertEquals(0, items.get(0).getPositions(1));
assertEquals(0, items.get(0).getPositions(2));
- assertEquals(1,
+ assertEquals(1,
items.get(0).getStatistics().getIntStatistics().getMinimum());
index = recordReader.readRowIndex(1, null, null).getRowGroupIndex();
assertEquals(3, index.length);
items = index[1].getEntryList();
- assertEquals(2,
+ assertEquals(2,
items.get(0).getStatistics().getIntStatistics().getMaximum());
}
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java?rev=1673599&r1=1673598&r2=1673599&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java Wed Apr 15 01:11:47 2015
@@ -52,7 +52,7 @@ public class TestVectorizer {
columns.add("col3");
//Generate vectorized expression
- vContext = new VectorizationContext(columns);
+ vContext = new VectorizationContext("name", columns);
}
@Description(name = "fake", value = "FAKE")