You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/27 00:30:43 UTC
svn commit: r1627904 - in /hive/trunk: itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/
ql/src/test/queries/clientpos...
Author: gunther
Date: Fri Sep 26 22:30:42 2014
New Revision: 1627904
URL: http://svn.apache.org/r1627904
Log:
HIVE-8171: Tez and Vectorized Reduce doesn't create scratch columns (Matt McCline via Gunther Hagleitner)
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/trunk/ql/src/test/queries/clientpositive/vector_char_simple.q
hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_simple.q
hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_simple.q.out
hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_simple.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_char_simple.q.out
hive/trunk/ql/src/test/results/clientpositive/vector_varchar_simple.q.out
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Fri Sep 26 22:30:42 2014
@@ -154,11 +154,13 @@ minitez.query.files.shared=alter_merge_2
update_where_partitioned.q,\
update_two_cols.q,\
vector_cast_constant.q,\
+ vector_char_simple.q,\
vector_data_types.q,\
vector_decimal_aggregate.q,\
vector_left_outer_join.q,\
vector_mapjoin_reduce.q,\
vector_string_concat.q,\
+ vector_varchar_simple.q,\
vectorization_0.q,\
vectorization_12.q,\
vectorization_13.q,\
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java Fri Sep 26 22:30:42 2014
@@ -100,7 +100,8 @@ public class ReduceRecordProcessor exte
sources[tag] = new ReduceRecordSource();
sources[tag].init(jconf, reducer, redWork.getVectorMode(), keyTableDesc, valueTableDesc,
- reader, tag == position, (byte) tag);
+ reader, tag == position, (byte) tag,
+ redWork.getScratchColumnVectorTypes());
ois[tag] = sources[tag].getObjectInspector();
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java Fri Sep 26 22:30:42 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.t
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.Op
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
import org.apache.hadoop.hive.ql.log.PerfLogger;
@@ -85,6 +87,7 @@ public class ReduceRecordSource implemen
List<Object> row = new ArrayList<Object>(Utilities.reduceFieldNameList.size());
private DataOutputBuffer buffer;
+ private VectorizedRowBatchCtx batchContext;
private VectorizedRowBatch batch;
// number of columns pertaining to keys in a vectorized row batch
@@ -110,7 +113,8 @@ public class ReduceRecordSource implemen
private final boolean grouped = true;
void init(JobConf jconf, Operator<?> reducer, boolean vectorized, TableDesc keyTableDesc,
- TableDesc valueTableDesc, KeyValuesReader reader, boolean handleGroupKey, byte tag)
+ TableDesc valueTableDesc, KeyValuesReader reader, boolean handleGroupKey, byte tag,
+ Map<String, Map<Integer, String>> scratchColumnVectorTypes)
throws Exception {
ObjectInspector keyObjectInspector;
@@ -149,9 +153,6 @@ public class ReduceRecordSource implemen
/* vectorization only works with struct object inspectors */
valueStructInspectors = (StructObjectInspector) valueObjectInspector;
- batch = VectorizedBatchUtil.constructVectorizedRowBatch(keyStructInspector,
- valueStructInspectors);
-
final int totalColumns = keysColumnOffset +
valueStructInspectors.getAllStructFieldRefs().size();
valueStringWriters = new ArrayList<VectorExpressionWriter>(totalColumns);
@@ -178,6 +179,12 @@ public class ReduceRecordSource implemen
ois.add(field.getFieldObjectInspector());
}
rowObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(colNames, ois);
+
+ Map<Integer, String> reduceShuffleScratchColumnTypeMap =
+ scratchColumnVectorTypes.get("_REDUCE_SHUFFLE_");
+ batchContext = new VectorizedRowBatchCtx();
+ batchContext.init(reduceShuffleScratchColumnTypeMap, (StructObjectInspector) rowObjectInspector);
+ batch = batchContext.createVectorizedRowBatch();
} else {
ois.add(keyObjectInspector);
ois.add(valueObjectInspector);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Fri Sep 26 22:30:42 2014
@@ -140,6 +140,20 @@ public class VectorizedRowBatchCtx {
/**
+ * Initializes the VectorizedRowBatch context based on an scratch column type map and
+ * object inspector.
+ * @param columnTypeMap
+ * @param rowOI
+ * Object inspector that shapes the column types
+ */
+ public void init(Map<Integer, String> columnTypeMap,
+ StructObjectInspector rowOI) {
+ this.columnTypeMap = columnTypeMap;
+ this.rowOI= rowOI;
+ this.rawRowOI = rowOI;
+ }
+
+ /**
* Initializes VectorizedRowBatch context based on the
* split and Hive configuration (Job conf with hive Plan).
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri Sep 26 22:30:42 2014
@@ -411,10 +411,12 @@ public class Vectorizer implements Physi
// Check value ObjectInspector.
ObjectInspector valueObjectInspector = reduceWork.getValueObjectInspector();
- if (valueObjectInspector == null || !(valueObjectInspector instanceof StructObjectInspector)) {
+ if (valueObjectInspector == null ||
+ !(valueObjectInspector instanceof StructObjectInspector)) {
return false;
}
- StructObjectInspector valueStructObjectInspector = (StructObjectInspector)valueObjectInspector;
+ StructObjectInspector valueStructObjectInspector =
+ (StructObjectInspector)valueObjectInspector;
valueColCount = valueStructObjectInspector.getAllStructFieldRefs().size();
} catch (Exception e) {
throw new SemanticException(e);
@@ -460,18 +462,20 @@ public class Vectorizer implements Physi
LOG.info("Vectorizing ReduceWork...");
reduceWork.setVectorMode(true);
- // For some reason, the DefaultGraphWalker does not descend down from the reducer Operator as expected.
- // We need to descend down, otherwise it breaks our algorithm that determines VectorizationContext...
- // Do we use PreOrderWalker instead of DefaultGraphWalker.
+ // For some reason, the DefaultGraphWalker does not descend down from the reducer Operator as
+ // expected. We need to descend down, otherwise it breaks our algorithm that determines
+ // VectorizationContext... Do we use PreOrderWalker instead of DefaultGraphWalker.
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
- ReduceWorkVectorizationNodeProcessor vnp = new ReduceWorkVectorizationNodeProcessor(reduceWork, keyColCount, valueColCount);
+ ReduceWorkVectorizationNodeProcessor vnp =
+ new ReduceWorkVectorizationNodeProcessor(reduceWork, keyColCount, valueColCount);
addReduceWorkRules(opRules, vnp);
Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
GraphWalker ogw = new PreOrderWalker(disp);
// iterator the reduce operator tree
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.add(reduceWork.getReducer());
- LOG.info("vectorizeReduceWork reducer Operator: " + reduceWork.getReducer().getName() + "...");
+ LOG.info("vectorizeReduceWork reducer Operator: " +
+ reduceWork.getReducer().getName() + "...");
HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutput);
@@ -550,7 +554,7 @@ public class Vectorizer implements Physi
protected final Map<String, VectorizationContext> scratchColumnContext =
new HashMap<String, VectorizationContext>();
- protected final Map<Operator<? extends OperatorDesc>, VectorizationContext> vContextsByTSOp =
+ protected final Map<Operator<? extends OperatorDesc>, VectorizationContext> vContextsByOp =
new HashMap<Operator<? extends OperatorDesc>, VectorizationContext>();
protected final Set<Operator<? extends OperatorDesc>> opsDone =
@@ -578,28 +582,30 @@ public class Vectorizer implements Physi
return scratchColumnMap;
}
- public VectorizationContext walkStackToFindVectorizationContext(Stack<Node> stack, Operator<? extends OperatorDesc> op)
- throws SemanticException {
+ public VectorizationContext walkStackToFindVectorizationContext(Stack<Node> stack,
+ Operator<? extends OperatorDesc> op) throws SemanticException {
VectorizationContext vContext = null;
if (stack.size() <= 1) {
- throw new SemanticException(String.format("Expected operator stack for operator %s to have at least 2 operators", op.getName()));
+ throw new SemanticException(
+ String.format("Expected operator stack for operator %s to have at least 2 operators",
+ op.getName()));
}
// Walk down the stack of operators until we found one willing to give us a context.
// At the bottom will be the root operator, guaranteed to have a context
int i= stack.size()-2;
while (vContext == null) {
if (i < 0) {
- throw new SemanticException(String.format("Did not find vectorization context for operator %s in operator stack", op.getName()));
+ return null;
}
Operator<? extends OperatorDesc> opParent = (Operator<? extends OperatorDesc>) stack.get(i);
- vContext = vContextsByTSOp.get(opParent);
+ vContext = vContextsByOp.get(opParent);
--i;
}
return vContext;
}
- public Operator<? extends OperatorDesc> doVectorize(Operator<? extends OperatorDesc> op, VectorizationContext vContext)
- throws SemanticException {
+ public Operator<? extends OperatorDesc> doVectorize(Operator<? extends OperatorDesc> op,
+ VectorizationContext vContext) throws SemanticException {
Operator<? extends OperatorDesc> vectorOp = op;
try {
if (!opsDone.contains(op)) {
@@ -611,7 +617,7 @@ public class Vectorizer implements Physi
if (vectorOp instanceof VectorizationContextRegion) {
VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
- vContextsByTSOp.put(op, vOutContext);
+ vContextsByOp.put(op, vOutContext);
scratchColumnContext.put(vOutContext.getFileKey(), vOutContext);
}
}
@@ -658,13 +664,24 @@ public class Vectorizer implements Physi
//
vContext.setFileKey(onefile);
scratchColumnContext.put(onefile, vContext);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vectorized MapWork operator " + op.getName() +
+ " with vectorization context key=" + vContext.getFileKey() +
+ ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vContext.getColumnMap().toString());
+ }
break;
}
}
}
- vContextsByTSOp.put(op, vContext);
+ vContextsByOp.put(op, vContext);
} else {
vContext = walkStackToFindVectorizationContext(stack, op);
+ if (vContext == null) {
+ throw new SemanticException(
+ String.format("Did not find vectorization context for operator %s in operator stack",
+ op.getName()));
+ }
}
assert vContext != null;
@@ -679,7 +696,22 @@ public class Vectorizer implements Physi
return null;
}
- doVectorize(op, vContext);
+ Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vectorized MapWork operator " + vectorOp.getName() +
+ " with vectorization context key=" + vContext.getFileKey() +
+ ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vContext.getColumnMap().toString());
+ if (vectorOp instanceof VectorizationContextRegion) {
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+ VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
+ LOG.debug("Vectorized MapWork operator " + vectorOp.getName() +
+ " added new vectorization context key=" + vOutContext.getFileKey() +
+ ", vectorTypes: " + vOutContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vOutContext.getColumnMap().toString());
+ }
+ }
return null;
}
@@ -691,6 +723,8 @@ public class Vectorizer implements Physi
private int keyColCount;
private int valueColCount;
private Map<String, Integer> reduceColumnNameMap;
+
+ private VectorizationContext reduceShuffleVectorizationContext;
private Operator<? extends OperatorDesc> rootVectorOp;
@@ -698,12 +732,14 @@ public class Vectorizer implements Physi
return rootVectorOp;
}
- public ReduceWorkVectorizationNodeProcessor(ReduceWork rWork, int keyColCount, int valueColCount) {
+ public ReduceWorkVectorizationNodeProcessor(ReduceWork rWork, int keyColCount,
+ int valueColCount) {
this.rWork = rWork;
reduceColumnNameMap = rWork.getReduceColumnNameMap();
this.keyColCount = keyColCount;
this.valueColCount = valueColCount;
rootVectorOp = null;
+ reduceShuffleVectorizationContext = null;
}
@Override
@@ -711,7 +747,8 @@ public class Vectorizer implements Physi
Object... nodeOutputs) throws SemanticException {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
- LOG.info("ReduceWorkVectorizationNodeProcessor processing Operator: " + op.getName() + "...");
+ LOG.info("ReduceWorkVectorizationNodeProcessor processing Operator: " +
+ op.getName() + "...");
VectorizationContext vContext = null;
@@ -719,10 +756,24 @@ public class Vectorizer implements Physi
if (op.getParentOperators().size() == 0) {
vContext = getReduceVectorizationContext(reduceColumnNameMap);
- vContextsByTSOp.put(op, vContext);
+ vContext.setFileKey("_REDUCE_SHUFFLE_");
+ scratchColumnContext.put("_REDUCE_SHUFFLE_", vContext);
+ reduceShuffleVectorizationContext = vContext;
saveRootVectorOp = true;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context key=" +
+ vContext.getFileKey() +
+ ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vContext.getColumnMap().toString());
+ }
} else {
vContext = walkStackToFindVectorizationContext(stack, op);
+ if (vContext == null) {
+ // If we didn't find a context among the operators, assume the top -- reduce shuffle's
+ // vectorization context.
+ vContext = reduceShuffleVectorizationContext;
+ }
}
assert vContext != null;
@@ -738,6 +789,21 @@ public class Vectorizer implements Physi
}
Operator<? extends OperatorDesc> vectorOp = doVectorize(op, vContext);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() +
+ " with vectorization context key=" + vContext.getFileKey() +
+ ", vectorTypes: " + vContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vContext.getColumnMap().toString());
+ if (vectorOp instanceof VectorizationContextRegion) {
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+ VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
+ LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() +
+ " added new vectorization context key=" + vOutContext.getFileKey() +
+ ", vectorTypes: " + vOutContext.getOutputColumnTypeMap().toString() +
+ ", columnMap: " + vOutContext.getColumnMap().toString());
+ }
+ }
if (vectorOp instanceof VectorGroupByOperator) {
VectorGroupByOperator groupBy = (VectorGroupByOperator) vectorOp;
VectorGroupByDesc vectorDesc = groupBy.getConf().getVectorDesc();
@@ -791,7 +857,6 @@ public class Vectorizer implements Physi
boolean validateMapWorkOperator(Operator<? extends OperatorDesc> op, boolean isTez) {
boolean ret = false;
- LOG.info("Validating MapWork operator " + op.getType().name());
switch (op.getType()) {
case MAPJOIN:
if (op instanceof MapJoinOperator) {
@@ -829,7 +894,6 @@ public class Vectorizer implements Physi
boolean validateReduceWorkOperator(Operator<? extends OperatorDesc> op) {
boolean ret = false;
- LOG.info("Validating ReduceWork operator " + op.getType().name());
switch (op.getType()) {
case EXTRACT:
ret = validateExtractOperator((ExtractOperator) op);
@@ -843,12 +907,7 @@ public class Vectorizer implements Physi
}
break;
case GROUPBY:
- if (HiveConf.getBoolVar(physicalContext.getConf(),
- HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED)) {
- ret = validateGroupByOperator((GroupByOperator) op, true, true);
- } else {
- ret = false;
- }
+ ret = validateGroupByOperator((GroupByOperator) op, true, true);
break;
case FILTER:
ret = validateFilterOperator((FilterOperator) op);
@@ -1015,7 +1074,8 @@ public class Vectorizer implements Physi
return validateExprNodeDesc(descs, VectorExpressionDescriptor.Mode.PROJECTION);
}
- private boolean validateExprNodeDesc(List<ExprNodeDesc> descs, VectorExpressionDescriptor.Mode mode) {
+ private boolean validateExprNodeDesc(List<ExprNodeDesc> descs,
+ VectorExpressionDescriptor.Mode mode) {
for (ExprNodeDesc d : descs) {
boolean ret = validateExprNodeDesc(d, mode);
if (!ret) {
@@ -1080,11 +1140,11 @@ public class Vectorizer implements Physi
VectorizationContext vc = new ValidatorVectorizationContext();
if (vc.getVectorExpression(desc, mode) == null) {
// TODO: this cannot happen - VectorizationContext throws in such cases.
- LOG.debug("getVectorExpression returned null");
+ LOG.info("getVectorExpression returned null");
return false;
}
} catch (Exception e) {
- LOG.debug("Failed to vectorize", e);
+ LOG.info("Failed to vectorize", e);
return false;
}
return true;
@@ -1115,11 +1175,11 @@ public class Vectorizer implements Physi
VectorizationContext vc = new ValidatorVectorizationContext();
if (vc.getAggregatorExpression(aggDesc, isReduce) == null) {
// TODO: this cannot happen - VectorizationContext throws in such cases.
- LOG.debug("getAggregatorExpression returned null");
+ LOG.info("getAggregatorExpression returned null");
return false;
}
} catch (Exception e) {
- LOG.debug("Failed to vectorize", e);
+ LOG.info("Failed to vectorize", e);
return false;
}
return true;
@@ -1173,11 +1233,13 @@ public class Vectorizer implements Physi
return new VectorizationContext(cmap, columnCount);
}
- private VectorizationContext getReduceVectorizationContext(Map<String, Integer> reduceColumnNameMap) {
+ private VectorizationContext getReduceVectorizationContext(
+ Map<String, Integer> reduceColumnNameMap) {
return new VectorizationContext(reduceColumnNameMap, reduceColumnNameMap.size());
}
- private void fixupParentChildOperators(Operator<? extends OperatorDesc> op, Operator<? extends OperatorDesc> vectorOp) {
+ private void fixupParentChildOperators(Operator<? extends OperatorDesc> op,
+ Operator<? extends OperatorDesc> vectorOp) {
if (op.getParentOperators() != null) {
vectorOp.setParentOperators(op.getParentOperators());
for (Operator<? extends OperatorDesc> p : op.getParentOperators()) {
Modified: hive/trunk/ql/src/test/queries/clientpositive/vector_char_simple.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_char_simple.q?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_char_simple.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_char_simple.q Fri Sep 26 22:30:42 2014
@@ -41,3 +41,16 @@ order by key desc
limit 5;
drop table char_2;
+
+
+-- Implicit conversion. Occurs in reduce-side under Tez.
+create table char_3 (
+ field char(12)
+) stored as orc;
+
+explain
+insert into table char_3 select cint from alltypesorc limit 10;
+
+insert into table char_3 select cint from alltypesorc limit 10;
+
+drop table char_3;
Modified: hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_simple.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_simple.q?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_simple.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_varchar_simple.q Fri Sep 26 22:30:42 2014
@@ -1,12 +1,12 @@
SET hive.vectorized.execution.enabled=true;
-drop table char_2;
+drop table varchar_2;
-create table char_2 (
+create table varchar_2 (
key varchar(10),
value varchar(20)
) stored as orc;
-insert overwrite table char_2 select * from src;
+insert overwrite table varchar_2 select * from src;
select key, value
from src
@@ -14,13 +14,13 @@ order by key asc
limit 5;
explain select key, value
-from char_2
+from varchar_2
order by key asc
limit 5;
-- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key asc
limit 5;
@@ -30,14 +30,26 @@ order by key desc
limit 5;
explain select key, value
-from char_2
+from varchar_2
order by key desc
limit 5;
-- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key desc
limit 5;
-drop table char_2;
+drop table varchar_2;
+
+-- Implicit conversion. Occurs in reduce-side under Tez.
+create table varchar_3 (
+ field varchar(25)
+) stored as orc;
+
+explain
+insert into table varchar_3 select cint from alltypesorc limit 10;
+
+insert into table varchar_3 select cint from alltypesorc limit 10;
+
+drop table varchar_3;
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_simple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_simple.q.out?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_simple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/vector_char_simple.q.out Fri Sep 26 22:30:42 2014
@@ -234,3 +234,109 @@ POSTHOOK: query: drop table char_2
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@char_2
POSTHOOK: Output: default@char_2
+PREHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table char_3 (
+ field char(12)
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@char_3
+POSTHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table char_3 (
+ field char(12)
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@char_3
+PREHOOK: query: explain
+insert into table char_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert into table char_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: CAST( _col0 AS CHAR(12) (type: char(12))
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.char_3
+ Execution mode: vectorized
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.char_3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table char_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@char_3
+POSTHOOK: query: insert into table char_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@char_3
+POSTHOOK: Lineage: char_3.field EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+PREHOOK: query: drop table char_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@char_3
+PREHOOK: Output: default@char_3
+POSTHOOK: query: drop table char_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@char_3
+POSTHOOK: Output: default@char_3
Modified: hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_simple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_simple.q.out?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_simple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/vector_varchar_simple.q.out Fri Sep 26 22:30:42 2014
@@ -1,31 +1,31 @@
-PREHOOK: query: drop table char_2
+PREHOOK: query: drop table varchar_2
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table char_2
+POSTHOOK: query: drop table varchar_2
POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table char_2 (
+PREHOOK: query: create table varchar_2 (
key varchar(10),
value varchar(20)
) stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@char_2
-POSTHOOK: query: create table char_2 (
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: create table varchar_2 (
key varchar(10),
value varchar(20)
) stored as orc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@char_2
-PREHOOK: query: insert overwrite table char_2 select * from src
+POSTHOOK: Output: default@varchar_2
+PREHOOK: query: insert overwrite table varchar_2 select * from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: default@char_2
-POSTHOOK: query: insert overwrite table char_2 select * from src
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: insert overwrite table varchar_2 select * from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: default@char_2
-POSTHOOK: Lineage: char_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: char_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Output: default@varchar_2
+POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select key, value
from src
order by key asc
@@ -46,12 +46,12 @@ POSTHOOK: Input: default@src
10 val_10
100 val_100
PREHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
PREHOOK: type: QUERY
POSTHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
POSTHOOK: type: QUERY
@@ -69,7 +69,7 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: char_2
+ alias: varchar_2
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: varchar(10)), value (type: varchar(20))
@@ -107,19 +107,19 @@ STAGE PLANS:
PREHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
PREHOOK: type: QUERY
-PREHOOK: Input: default@char_2
+PREHOOK: Input: default@varchar_2
#### A masked pattern was here ####
POSTHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_2
+POSTHOOK: Input: default@varchar_2
#### A masked pattern was here ####
0 val_0
0 val_0
@@ -146,12 +146,12 @@ POSTHOOK: Input: default@src
97 val_97
96 val_96
PREHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
PREHOOK: type: QUERY
POSTHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
POSTHOOK: type: QUERY
@@ -169,7 +169,7 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: char_2
+ alias: varchar_2
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: varchar(10)), value (type: varchar(20))
@@ -207,30 +207,136 @@ STAGE PLANS:
PREHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
PREHOOK: type: QUERY
-PREHOOK: Input: default@char_2
+PREHOOK: Input: default@varchar_2
#### A masked pattern was here ####
POSTHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_2
+POSTHOOK: Input: default@varchar_2
#### A masked pattern was here ####
98 val_98
98 val_98
97 val_97
97 val_97
96 val_96
-PREHOOK: query: drop table char_2
+PREHOOK: query: drop table varchar_2
PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@char_2
-PREHOOK: Output: default@char_2
-POSTHOOK: query: drop table char_2
+PREHOOK: Input: default@varchar_2
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: drop table varchar_2
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@char_2
-POSTHOOK: Output: default@char_2
+POSTHOOK: Input: default@varchar_2
+POSTHOOK: Output: default@varchar_2
+PREHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table varchar_3 (
+ field varchar(25)
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table varchar_3 (
+ field varchar(25)
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@varchar_3
+PREHOOK: query: explain
+insert into table varchar_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert into table varchar_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: CAST( _col0 AS varchar(25)) (type: varchar(25))
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.varchar_3
+ Execution mode: vectorized
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.varchar_3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@varchar_3
+POSTHOOK: Lineage: varchar_3.field EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+PREHOOK: query: drop table varchar_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@varchar_3
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: drop table varchar_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@varchar_3
+POSTHOOK: Output: default@varchar_3
Modified: hive/trunk/ql/src/test/results/clientpositive/vector_char_simple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_char_simple.q.out?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_char_simple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_char_simple.q.out Fri Sep 26 22:30:42 2014
@@ -220,3 +220,98 @@ POSTHOOK: query: drop table char_2
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@char_2
POSTHOOK: Output: default@char_2
+PREHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table char_3 (
+ field char(12)
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@char_3
+POSTHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table char_3 (
+ field char(12)
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@char_3
+PREHOOK: query: explain
+insert into table char_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert into table char_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: CAST( _col0 AS CHAR(12) (type: char(12))
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.char_3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.char_3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table char_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@char_3
+POSTHOOK: query: insert into table char_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@char_3
+POSTHOOK: Lineage: char_3.field EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+PREHOOK: query: drop table char_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@char_3
+PREHOOK: Output: default@char_3
+POSTHOOK: query: drop table char_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@char_3
+POSTHOOK: Output: default@char_3
Modified: hive/trunk/ql/src/test/results/clientpositive/vector_varchar_simple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vector_varchar_simple.q.out?rev=1627904&r1=1627903&r2=1627904&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vector_varchar_simple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/vector_varchar_simple.q.out Fri Sep 26 22:30:42 2014
@@ -1,31 +1,31 @@
-PREHOOK: query: drop table char_2
+PREHOOK: query: drop table varchar_2
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table char_2
+POSTHOOK: query: drop table varchar_2
POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table char_2 (
+PREHOOK: query: create table varchar_2 (
key varchar(10),
value varchar(20)
) stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
-PREHOOK: Output: default@char_2
-POSTHOOK: query: create table char_2 (
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: create table varchar_2 (
key varchar(10),
value varchar(20)
) stored as orc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
-POSTHOOK: Output: default@char_2
-PREHOOK: query: insert overwrite table char_2 select * from src
+POSTHOOK: Output: default@varchar_2
+PREHOOK: query: insert overwrite table varchar_2 select * from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: default@char_2
-POSTHOOK: query: insert overwrite table char_2 select * from src
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: insert overwrite table varchar_2 select * from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: default@char_2
-POSTHOOK: Lineage: char_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: char_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Output: default@varchar_2
+POSTHOOK: Lineage: varchar_2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: varchar_2.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select key, value
from src
order by key asc
@@ -46,12 +46,12 @@ POSTHOOK: Input: default@src
10 val_10
100 val_100
PREHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
PREHOOK: type: QUERY
POSTHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
POSTHOOK: type: QUERY
@@ -64,7 +64,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: char_2
+ alias: varchar_2
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: varchar(10)), value (type: varchar(20))
@@ -100,19 +100,19 @@ STAGE PLANS:
PREHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
PREHOOK: type: QUERY
-PREHOOK: Input: default@char_2
+PREHOOK: Input: default@varchar_2
#### A masked pattern was here ####
POSTHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key asc
limit 5
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_2
+POSTHOOK: Input: default@varchar_2
#### A masked pattern was here ####
0 val_0
0 val_0
@@ -139,12 +139,12 @@ POSTHOOK: Input: default@src
97 val_97
96 val_96
PREHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
PREHOOK: type: QUERY
POSTHOOK: query: explain select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
POSTHOOK: type: QUERY
@@ -157,7 +157,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: char_2
+ alias: varchar_2
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: varchar(10)), value (type: varchar(20))
@@ -193,30 +193,125 @@ STAGE PLANS:
PREHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
PREHOOK: type: QUERY
-PREHOOK: Input: default@char_2
+PREHOOK: Input: default@varchar_2
#### A masked pattern was here ####
POSTHOOK: query: -- should match the query from src
select key, value
-from char_2
+from varchar_2
order by key desc
limit 5
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_2
+POSTHOOK: Input: default@varchar_2
#### A masked pattern was here ####
98 val_98
98 val_98
97 val_97
97 val_97
96 val_96
-PREHOOK: query: drop table char_2
+PREHOOK: query: drop table varchar_2
PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@char_2
-PREHOOK: Output: default@char_2
-POSTHOOK: query: drop table char_2
+PREHOOK: Input: default@varchar_2
+PREHOOK: Output: default@varchar_2
+POSTHOOK: query: drop table varchar_2
POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@char_2
-POSTHOOK: Output: default@char_2
+POSTHOOK: Input: default@varchar_2
+POSTHOOK: Output: default@varchar_2
+PREHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table varchar_3 (
+ field varchar(25)
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: -- Implicit conversion. Occurs in reduce-side under Tez.
+create table varchar_3 (
+ field varchar(25)
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@varchar_3
+PREHOOK: query: explain
+insert into table varchar_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert into table varchar_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: CAST( _col0 AS varchar(25)) (type: varchar(25))
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.varchar_3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.varchar_3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@varchar_3
+POSTHOOK: Lineage: varchar_3.field EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+PREHOOK: query: drop table varchar_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@varchar_3
+PREHOOK: Output: default@varchar_3
+POSTHOOK: query: drop table varchar_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@varchar_3
+POSTHOOK: Output: default@varchar_3