You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/08/25 16:21:39 UTC
[2/5] hive git commit: HIVE-20315: Vectorization: Fix more NULL /
Wrong Results issues and avoid unnecessary casts/conversions (Matt McCline,
reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStructField.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStructField.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStructField.java
new file mode 100644
index 0000000..5062997
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStructField.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import java.lang.reflect.Constructor;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow;
+import org.apache.hadoop.hive.ql.exec.vector.VectorRandomBatchSource;
+import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
+import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.GenerationSpec;
+import org.apache.hadoop.hive.ql.exec.vector.VectorRandomRowSource.SupportedTypes;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNegative;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+import junit.framework.Assert;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestVectorStructField {
+
+ @Test
+ public void testStructField() throws Exception {
+ Random random = new Random(7743);
+
+ for (int i = 0; i < 5; i++) {
+ doStructFieldTests(random);
+ }
+ }
+
+ public enum StructFieldTestMode {
+ ROW_MODE,
+ VECTOR_EXPRESSION;
+
+ static final int count = values().length;
+ }
+
+ private void doStructFieldTests(Random random) throws Exception {
+ String structTypeName =
+ VectorRandomRowSource.getDecoratedTypeName(
+ random, "struct", SupportedTypes.ALL, /* allowedTypeNameSet */ null,
+ /* depth */ 0, /* maxDepth */ 2);
+ StructTypeInfo structTypeInfo =
+ (StructTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(structTypeName);
+
+ List<String> fieldNameList = structTypeInfo.getAllStructFieldNames();
+ final int fieldCount = fieldNameList.size();
+ for (int fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
+ doOneStructFieldTest(random, structTypeInfo, structTypeName, fieldIndex);
+ }
+ }
+
+ private void doOneStructFieldTest(Random random, StructTypeInfo structTypeInfo,
+ String structTypeName, int fieldIndex)
+ throws Exception {
+
+ List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
+ List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList =
+ new ArrayList<DataTypePhysicalVariation>();
+
+ List<String> columns = new ArrayList<String>();
+ int columnNum = 1;
+
+ generationSpecList.add(
+ GenerationSpec.createSameType(structTypeInfo));
+ explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
+
+ ExprNodeDesc col1Expr;
+ String columnName = "col" + (columnNum++);
+ col1Expr = new ExprNodeColumnDesc(structTypeInfo, columnName, "table", false);
+ columns.add(columnName);
+
+ ObjectInspector structObjectInspector =
+ VectorRandomRowSource.getObjectInspector(structTypeInfo);
+ List<ObjectInspector> objectInspectorList = new ArrayList<ObjectInspector>();
+ objectInspectorList.add(structObjectInspector);
+
+ List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
+ children.add(col1Expr);
+
+ //----------------------------------------------------------------------------------------------
+
+ String[] columnNames = columns.toArray(new String[0]);
+
+ VectorRandomRowSource rowSource = new VectorRandomRowSource();
+
+ rowSource.initGenerationSpecSchema(
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
+ explicitDataTypePhysicalVariationList);
+
+ Object[][] randomRows = rowSource.randomRows(100000);
+
+ VectorRandomBatchSource batchSource =
+ VectorRandomBatchSource.createInterestingBatches(
+ random,
+ rowSource,
+ randomRows,
+ null);
+
+ List<String> fieldNameList = structTypeInfo.getAllStructFieldNames();
+ List<TypeInfo> fieldTypeInfoList = structTypeInfo.getAllStructFieldTypeInfos();
+
+ String randomFieldName = fieldNameList.get(fieldIndex);
+ TypeInfo outputTypeInfo = fieldTypeInfoList.get(fieldIndex);
+
+ ExprNodeFieldDesc exprNodeFieldDesc =
+ new ExprNodeFieldDesc(outputTypeInfo, col1Expr, randomFieldName, /* isList */ false);
+
+ final int rowCount = randomRows.length;
+ Object[][] resultObjectsArray = new Object[StructFieldTestMode.count][];
+ for (int i = 0; i < StructFieldTestMode.count; i++) {
+
+ Object[] resultObjects = new Object[rowCount];
+ resultObjectsArray[i] = resultObjects;
+
+ StructFieldTestMode negativeTestMode = StructFieldTestMode.values()[i];
+ switch (negativeTestMode) {
+ case ROW_MODE:
+ doRowStructFieldTest(
+ structTypeInfo,
+ columns,
+ children,
+ exprNodeFieldDesc,
+ randomRows,
+ rowSource.rowStructObjectInspector(),
+ outputTypeInfo,
+ resultObjects);
+ break;
+ case VECTOR_EXPRESSION:
+ doVectorStructFieldTest(
+ structTypeInfo,
+ columns,
+ columnNames,
+ rowSource.typeInfos(),
+ rowSource.dataTypePhysicalVariations(),
+ children,
+ exprNodeFieldDesc,
+ negativeTestMode,
+ batchSource,
+ exprNodeFieldDesc.getWritableObjectInspector(),
+ outputTypeInfo,
+ resultObjects);
+ break;
+ default:
+ throw new RuntimeException("Unexpected Negative operator test mode " + negativeTestMode);
+ }
+ }
+
+ for (int i = 0; i < rowCount; i++) {
+ // Row-mode is the expected value.
+ Object expectedResult = resultObjectsArray[0][i];
+
+ for (int v = 1; v < StructFieldTestMode.count; v++) {
+ Object vectorResult = resultObjectsArray[v][i];
+ if (expectedResult == null || vectorResult == null) {
+ if (expectedResult != null || vectorResult != null) {
+ Assert.fail(
+ "Row " + i +
+ " structTypeName " + structTypeName +
+ " outputTypeName " + outputTypeInfo.getTypeName() +
+ " " + StructFieldTestMode.values()[v] +
+ " result is NULL " + (vectorResult == null) +
+ " does not match row-mode expected result is NULL " + (expectedResult == null) +
+ " row values " + Arrays.toString(randomRows[i]));
+ }
+ } else {
+
+ if (!expectedResult.equals(vectorResult)) {
+ Assert.fail(
+ "Row " + i +
+ " structTypeName " + structTypeName +
+ " outputTypeName " + outputTypeInfo.getTypeName() +
+ " " + StructFieldTestMode.values()[v] +
+ " result " + vectorResult.toString() +
+ " (" + vectorResult.getClass().getSimpleName() + ")" +
+ " does not match row-mode expected result " + expectedResult.toString() +
+ " (" + expectedResult.getClass().getSimpleName() + ")" +
+ " row values " + Arrays.toString(randomRows[i]));
+ }
+ }
+ }
+ }
+ }
+
+ private void doRowStructFieldTest(TypeInfo typeInfo,
+ List<String> columns, List<ExprNodeDesc> children,
+ ExprNodeFieldDesc exprNodeFieldDesc,
+ Object[][] randomRows,
+ ObjectInspector rowInspector,
+ TypeInfo outputTypeInfo, Object[] resultObjects) throws Exception {
+
+ /*
+ System.out.println(
+ "*DEBUG* typeInfo " + typeInfo.toString() +
+ " negativeTestMode ROW_MODE" +
+ " exprDesc " + exprDesc.toString());
+ */
+
+ HiveConf hiveConf = new HiveConf();
+ ExprNodeEvaluator evaluator =
+ ExprNodeEvaluatorFactory.get(exprNodeFieldDesc, hiveConf);
+ evaluator.initialize(rowInspector);
+
+ ObjectInspector objectInspector =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(
+ outputTypeInfo);
+
+ final int rowCount = randomRows.length;
+ for (int i = 0; i < rowCount; i++) {
+ Object[] row = randomRows[i];
+ Object result = evaluator.evaluate(row);
+ Object copyResult = null;
+ try {
+ copyResult =
+ ObjectInspectorUtils.copyToStandardObject(
+ result, objectInspector, ObjectInspectorCopyOption.WRITABLE);
+ } catch (Exception e) {
+ System.out.println("here");
+ }
+ resultObjects[i] = copyResult;
+ }
+ }
+
+ private void extractResultObjects(VectorizedRowBatch batch, int rowIndex,
+ VectorExtractRow resultVectorExtractRow, Object[] scrqtchRow,
+ ObjectInspector objectInspector, Object[] resultObjects) {
+
+ boolean selectedInUse = batch.selectedInUse;
+ int[] selected = batch.selected;
+ for (int logicalIndex = 0; logicalIndex < batch.size; logicalIndex++) {
+ final int batchIndex = (selectedInUse ? selected[logicalIndex] : logicalIndex);
+ resultVectorExtractRow.extractRow(batch, batchIndex, scrqtchRow);
+
+ Object copyResult =
+ ObjectInspectorUtils.copyToStandardObject(
+ scrqtchRow[0], objectInspector, ObjectInspectorCopyOption.WRITABLE);
+ resultObjects[rowIndex++] = copyResult;
+ }
+ }
+
+ private void doVectorStructFieldTest(TypeInfo typeInfo,
+ List<String> columns,
+ String[] columnNames,
+ TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations,
+ List<ExprNodeDesc> children,
+ ExprNodeFieldDesc exprNodeFieldDesc,
+ StructFieldTestMode negativeTestMode,
+ VectorRandomBatchSource batchSource,
+ ObjectInspector objectInspector,
+ TypeInfo outputTypeInfo, Object[] resultObjects)
+ throws Exception {
+
+ HiveConf hiveConf = new HiveConf();
+
+ VectorizationContext vectorizationContext =
+ new VectorizationContext(
+ "name",
+ columns,
+ Arrays.asList(typeInfos),
+ Arrays.asList(dataTypePhysicalVariations),
+ hiveConf);
+ VectorExpression vectorExpression =
+ vectorizationContext.getVectorExpression(exprNodeFieldDesc);
+ vectorExpression.transientInit();
+
+ if (negativeTestMode == StructFieldTestMode.VECTOR_EXPRESSION &&
+ vectorExpression instanceof VectorUDFAdaptor) {
+ System.out.println(
+ "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() +
+ " negativeTestMode " + negativeTestMode +
+ " vectorExpression " + vectorExpression.toString());
+ }
+
+ String[] outputScratchTypeNames= vectorizationContext.getScratchColumnTypeNames();
+
+ VectorizedRowBatchCtx batchContext =
+ new VectorizedRowBatchCtx(
+ columnNames,
+ typeInfos,
+ dataTypePhysicalVariations,
+ /* dataColumnNums */ null,
+ /* partitionColumnCount */ 0,
+ /* virtualColumnCount */ 0,
+ /* neededVirtualColumns */ null,
+ outputScratchTypeNames,
+ null);
+
+ VectorizedRowBatch batch = batchContext.createVectorizedRowBatch();
+
+ VectorExtractRow resultVectorExtractRow = new VectorExtractRow();
+ resultVectorExtractRow.init(
+ new TypeInfo[] { outputTypeInfo }, new int[] { vectorExpression.getOutputColumnNum() });
+ Object[] scrqtchRow = new Object[1];
+
+ // System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName());
+
+ /*
+ System.out.println(
+ "*DEBUG* typeInfo " + typeInfo.toString() +
+ " negativeTestMode " + negativeTestMode +
+ " vectorExpression " + vectorExpression.toString());
+ */
+
+ batchSource.resetBatchIteration();
+ int rowIndex = 0;
+ while (true) {
+ if (!batchSource.fillNextBatch(batch)) {
+ break;
+ }
+ vectorExpression.evaluate(batch);
+ extractResultObjects(batch, rowIndex, resultVectorExtractRow, scrqtchRow,
+ objectInspector, resultObjects);
+ rowIndex += batch.size;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java
index a978782..2997dcd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorSubStr.java
@@ -118,7 +118,8 @@ public class TestVectorSubStr {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java
index c31bec5..21109f3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExtract.java
@@ -137,7 +137,8 @@ public class TestVectorTimestampExtract {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java
index 05a98a6..df91443 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastRowHashMap.java
@@ -144,8 +144,10 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap();
VectorRandomRowSource valueSource = new VectorRandomRowSource();
-
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -170,7 +172,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -194,8 +198,10 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VerifyFastRowHashMap verifyTable = new VerifyFastRowHashMap();
VectorRandomRowSource valueSource = new VectorRandomRowSource();
-
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -220,7 +226,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -245,8 +253,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
-
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -271,7 +280,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -296,7 +307,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -321,7 +334,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -346,7 +361,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -371,7 +388,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -396,7 +415,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -421,7 +442,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -447,7 +470,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -472,7 +497,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -497,7 +524,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -522,7 +551,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -547,7 +578,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -572,7 +605,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -597,7 +632,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -622,7 +659,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -647,7 +686,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -672,7 +713,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -697,7 +740,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
@@ -722,7 +767,9 @@ public class TestVectorMapJoinFastRowHashMap extends CommonFastHashTable {
VectorRandomRowSource valueSource = new VectorRandomRowSource();
- valueSource.init(random, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ valueSource.init(
+ random, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
int rowCount = 1000;
Object[][] rows = valueSource.randomRows(rowCount);
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/queries/clientpositive/query_result_fileformat.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/query_result_fileformat.q b/ql/src/test/queries/clientpositive/query_result_fileformat.q
index a4c63e1..a32f25f 100644
--- a/ql/src/test/queries/clientpositive/query_result_fileformat.q
+++ b/ql/src/test/queries/clientpositive/query_result_fileformat.q
@@ -7,7 +7,7 @@ http://asdf' value from src limit 1;
select * from nzhang_test1;
select count(*) from nzhang_test1;
-explain
+explain vectorization detail
select * from nzhang_test1 where key='key1';
select * from nzhang_test1 where key='key1';
@@ -18,7 +18,7 @@ select * from nzhang_test1;
select count(*) from nzhang_test1;
-explain
+explain vectorization detail
select * from nzhang_test1 where key='key1';
select * from nzhang_test1 where key='key1';
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
index 3ab6547..411d693 100644
--- a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
@@ -524,7 +524,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 40, 42, 45, 46]
- selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean
, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColSca
lar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_
64) -> 48:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col
7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 23:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 19:boolean) -> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 23:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 23:string) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExpr
NullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:int) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean
, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColSca
lar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 48:decimal(10,2)) -> 30:decimal
(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 19:boolean)
-> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExprNullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:int) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean,
val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -856,8 +856,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 63, 65, 67, 68, 69, 71, 75, 78, 81, 82]
- selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolea
n, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLes
sLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractD
oubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 62:boolean, null, col 84)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 62:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 84:decimal(10,2)) -> 63:decimal(10,2), IfExprColumnNull(col 64:boolean, col 85:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 64:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 85:decimal(10,2)) -> 65:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) ->
66:boolean) -> 67:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 68:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 66:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 61:string, val DELIVER IN PERSON)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 66:boolean) -> 69:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 70:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 61:string, val TAKE BACK RETURN)(children: CastStringGroupToString(col 13:varchar(20)) -> 61:string) -> 70:boolean) -> 71:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 72:boolean, col 73:timestampcol 74:timestamp)(chi
ldren: LongColGreaterLongScalar(col 1:int, val 30) -> 72:boolean, CastDateToTimestamp(col 12:date) -> 73:timestamp, CastDateToTimestamp(col 11:date) -> 74:timestamp) -> 75:timestamp, IfExprCondExprNull(col 76:boolean, col 77:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 76:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 77:int) -> 78:int, IfExprNullCondExpr(col 79:boolean, null, col 80:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 79:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 80:int) -> 81:int, IfExprLongScalarLongScalar(col 83:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 82:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 82:int) -> 83:boolean) -> 82:date
+ projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 62, 64, 66, 67, 68, 70, 74, 77, 80, 81]
+ selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolea
n, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLes
sLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractD
oubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 61:boolean, null, col 83)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 61:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 83:decimal(10,2)) -> 62:decimal(10,2), IfExprColumnNull(col 63:boolean, col 84:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 63:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 84:decimal(10,2)) -> 64:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 65:boolean) -> 66:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax
, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 65:boolean) -> 67:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 65:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 65:boolean) -> 68:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 69:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 69:boolean) -> 70:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 71:boolean, col 72:timestampcol 73:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 71:boolean, CastDateToTimestamp(col 12:date) -> 72:timestamp, CastDateToTimestamp(col 11:date) -> 73:timestamp) -> 74:timestamp, IfExprCondExprNull(col 75:boolean, col 76:int
, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 75:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 76:int) -> 77:int, IfExprNullCondExpr(col 78:boolean, null, col 79:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 78:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 79:int) -> 80:int, IfExprLongScalarLongScalar(col 82:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 81:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 81:int) -> 82:boolean) -> 81:date
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -885,7 +885,7 @@ STAGE PLANS:
includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14]
dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, string, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)]
+ scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
index 31b3807..4ae125b 100644
--- a/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_char_mapjoin1.q.out
@@ -514,7 +514,6 @@ STAGE PLANS:
Map-reduce partition columns: CAST( _col1 AS STRING) (type: string)
Reduce Sink Vectorization:
className: VectorReduceSinkStringOperator
- keyExpressions: CastStringGroupToString(col 1:char(10)) -> 3:string
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 3 Data size: 294 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/results/clientpositive/llap/vector_udf1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_udf1.q.out b/ql/src/test/results/clientpositive/llap/vector_udf1.q.out
index 2d043e7..5db37eb 100644
--- a/ql/src/test/results/clientpositive/llap/vector_udf1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_udf1.q.out
@@ -71,8 +71,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [9, 10, 14]
- selectExpressions: StringGroupConcatColCol(col 0:string, col 1:string) -> 9:string, StringGroupConcatColCol(col 2:varchar(10), col 3:varchar(20)) -> 10:varchar(30), StringGroupColEqualStringGroupColumn(col 11:string, col 13:string)(children: StringGroupConcatColCol(col 0:string, col 1:string) -> 11:string, CastStringGroupToString(col 12:varchar(30))(children: StringGroupConcatColCol(col 2:varchar(10), col 3:varchar(20)) -> 12:varchar(30)) -> 13:string) -> 14:boolean
+ projectedOutputColumnNums: [9, 10, 13]
+ selectExpressions: StringGroupConcatColCol(col 0:string, col 1:string) -> 9:string, StringGroupConcatColCol(col 2:varchar(10), col 3:varchar(20)) -> 10:varchar(30), StringGroupColEqualStringGroupColumn(col 11:string, col 12:string)(children: StringGroupConcatColCol(col 0:string, col 1:string) -> 11:string, StringGroupConcatColCol(col 2:varchar(10), col 3:varchar(20)) -> 12:varchar(30)) -> 13:boolean
Statistics: Num rows: 1 Data size: 302 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
@@ -106,7 +106,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3]
dataColumns: c1:string, c2:string, c3:varchar(10), c4:varchar(20), d1:string, d2:string, d3:varchar(10), d4:varchar(10)
partitionColumnCount: 0
- scratchColumnTypeNames: [string, string, string, string, string, bigint]
+ scratchColumnTypeNames: [string, string, string, string, bigint]
Stage: Stage-0
Fetch Operator
@@ -172,8 +172,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [9, 10, 14]
- selectExpressions: StringUpper(col 1:string) -> 9:string, StringUpper(col 3:varchar(20)) -> 10:varchar(20), StringGroupColEqualStringGroupColumn(col 11:string, col 13:string)(children: StringUpper(col 1:string) -> 11:string, CastStringGroupToString(col 12:varchar(20))(children: StringUpper(col 3:varchar(20)) -> 12:varchar(20)) -> 13:string) -> 14:boolean
+ projectedOutputColumnNums: [9, 10, 13]
+ selectExpressions: StringUpper(col 1:string) -> 9:string, StringUpper(col 3:varchar(20)) -> 10:varchar(20), StringGroupColEqualStringGroupColumn(col 11:string, col 12:string)(children: StringUpper(col 1:string) -> 11:string, StringUpper(col 3:varchar(20)) -> 12:varchar(20)) -> 13:boolean
Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
@@ -207,7 +207,7 @@ STAGE PLANS:
includeColumns: [1, 3]
dataColumns: c1:string, c2:string, c3:varchar(10), c4:varchar(20), d1:string, d2:string, d3:varchar(10), d4:varchar(10)
partitionColumnCount: 0
- scratchColumnTypeNames: [string, string, string, string, string, bigint]
+ scratchColumnTypeNames: [string, string, string, string, bigint]
Stage: Stage-0
Fetch Operator
@@ -273,8 +273,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [9, 10, 14]
- selectExpressions: StringLower(col 1:string) -> 9:string, StringLower(col 3:varchar(20)) -> 10:varchar(20), StringGroupColEqualStringGroupColumn(col 11:string, col 13:string)(children: StringLower(col 1:string) -> 11:string, CastStringGroupToString(col 12:varchar(20))(children: StringLower(col 3:varchar(20)) -> 12:varchar(20)) -> 13:string) -> 14:boolean
+ projectedOutputColumnNums: [9, 10, 13]
+ selectExpressions: StringLower(col 1:string) -> 9:string, StringLower(col 3:varchar(20)) -> 10:varchar(20), StringGroupColEqualStringGroupColumn(col 11:string, col 12:string)(children: StringLower(col 1:string) -> 11:string, StringLower(col 3:varchar(20)) -> 12:varchar(20)) -> 13:boolean
Statistics: Num rows: 1 Data size: 292 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 1
@@ -308,7 +308,7 @@ STAGE PLANS:
includeColumns: [1, 3]
dataColumns: c1:string, c2:string, c3:varchar(10), c4:varchar(20), d1:string, d2:string, d3:varchar(10), d4:varchar(10)
partitionColumnCount: 0
- scratchColumnTypeNames: [string, string, string, string, string, bigint]
+ scratchColumnTypeNames: [string, string, string, string, bigint]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out b/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out
index 6a72515..25dc151 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out
@@ -183,8 +183,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [13, 14, 15, 16, 17, 18, 10, 20, 19, 21, 0, 1, 2, 3, 22, 23, 10, 24, 25, 27, 28, 29, 30, 31, 32, 33, 34, 4, 5, 35, 36, 37, 38, 39, 5, 41, 43, 45, 47, 48, 49, 51, 54, 55, 8, 56, 57, 26, 58, 59, 60, 61, 62, 63, 64, 65, 6, 67, 68, 69, 70, 66, 73]
- selectExpressions: CastLongToBooleanViaLongToLong(col 0:tinyint) -> 13:boolean, CastLongToBooleanViaLongToLong(col 1:smallint) -> 14:boolean, CastLongToBooleanViaLongToLong(col 2:int) -> 15:boolean, CastLongToBooleanViaLongToLong(col 3:bigint) -> 16:boolean, CastDoubleToBooleanViaDoubleToLong(col 4:float) -> 17:boolean, CastDoubleToBooleanViaDoubleToLong(col 5:double) -> 18:boolean, CastLongToBooleanViaLongToLong(col 19:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 19:bigint) -> 20:boolean, CastTimestampToBoolean(col 8:timestamp) -> 19:boolean, CastStringToBoolean(col 6) -> 21:boolean, CastDoubleToLong(col 4:float) -> 22:int, CastDoubleToLong(col 5:double) -> 23:int, CastTimestampToLong(col 8:timestamp) -> 24:int, CastStringToLong(col 6:string) -> 25:int, CastStringToLong(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 27:int, CastDoubleToLong(col 4:float) -> 28:tinyint, CastDoubleToLo
ng(col 4:float) -> 29:smallint, CastDoubleToLong(col 4:float) -> 30:bigint, CastLongToDouble(col 0:tinyint) -> 31:double, CastLongToDouble(col 1:smallint) -> 32:double, CastLongToDouble(col 2:int) -> 33:double, CastLongToDouble(col 3:bigint) -> 34:double, CastLongToDouble(col 10:boolean) -> 35:double, CastTimestampToDouble(col 8:timestamp) -> 36:double, CastStringToDouble(col 6:string) -> 37:double, CastStringToDouble(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 38:double, CastLongToFloatViaLongToDouble(col 2:int) -> 39:float, CastMillisecondsLongToTimestamp(col 0:tinyint) -> 41:timestamp, CastMillisecondsLongToTimestamp(col 1:smallint) -> 43:timestamp, CastMillisecondsLongToTimestamp(col 2:int) -> 45:timestamp, CastMillisecondsLongToTimestamp(col 3:bigint) -> 47:timestamp, CastDoubleToTimestamp(col 4:float) -> 48:timestamp, CastDoubleToTimestamp(col 5:double) -> 49:timestamp, CastMillisecondsLongToTimestamp(col 10:boolean) -> 51
:timestamp, CastMillisecondsLongToTimestamp(col 52:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 52:bigint) -> 54:timestamp, CastDateToTimestamp(col 52:date)(children: CastTimestampToDate(col 8:timestamp) -> 52:date) -> 55:timestamp, CastStringToTimestamp(col 6:string) -> 56:timestamp, CastStringToTimestamp(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 57:timestamp, CastLongToString(col 0:tinyint) -> 26:string, CastLongToString(col 1:smallint) -> 58:string, CastLongToString(col 2:int) -> 59:string, CastLongToString(col 3:bigint) -> 60:string, CastFloatToString(col 4:float) -> 61:string, CastDoubleToString(col 5:double) -> 62:string, CastBooleanToStringViaLongToString(col 10:boolean) -> 63:string, CastLongToString(col 52:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 52:bigint) -> 64:string, CastTimestampToString(col 8:timestamp) -> 65:string, CastStringGroupToString(col 66:char(10))(child
ren: CastStringGroupToChar(col 6:string, maxLength 10) -> 66:char(10)) -> 67:string, CastStringGroupToString(col 66:varchar(10))(children: CastStringGroupToVarChar(col 6:string, maxLength 10) -> 66:varchar(10)) -> 68:string, CastLongToFloatViaLongToDouble(col 52:int)(children: CastDoubleToLong(col 4:float) -> 52:int) -> 69:float, CastLongToDouble(col 52:int)(children: LongColMultiplyLongScalar(col 2:int, val 2) -> 52:int) -> 70:double, CastDoubleToString(col 71:double)(children: FuncSinDoubleToDouble(col 4:float) -> 71:double) -> 66:string, DoubleColAddDoubleColumn(col 71:double, col 72:double)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 71:float, CastLongToDouble(col 10:boolean) -> 72:double) -> 73:double
+ projectedOutputColumnNums: [13, 14, 15, 16, 17, 18, 10, 20, 19, 21, 0, 1, 2, 3, 22, 23, 10, 24, 25, 27, 28, 29, 30, 31, 32, 33, 34, 4, 5, 35, 36, 37, 38, 39, 5, 41, 43, 45, 47, 48, 49, 51, 54, 55, 8, 56, 57, 26, 58, 59, 60, 61, 62, 63, 64, 65, 6, 66, 67, 68, 69, 71, 73]
+ selectExpressions: CastLongToBooleanViaLongToLong(col 0:tinyint) -> 13:boolean, CastLongToBooleanViaLongToLong(col 1:smallint) -> 14:boolean, CastLongToBooleanViaLongToLong(col 2:int) -> 15:boolean, CastLongToBooleanViaLongToLong(col 3:bigint) -> 16:boolean, CastDoubleToBooleanViaDoubleToLong(col 4:float) -> 17:boolean, CastDoubleToBooleanViaDoubleToLong(col 5:double) -> 18:boolean, CastLongToBooleanViaLongToLong(col 19:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 19:bigint) -> 20:boolean, CastTimestampToBoolean(col 8:timestamp) -> 19:boolean, CastStringToBoolean(col 6) -> 21:boolean, CastDoubleToLong(col 4:float) -> 22:int, CastDoubleToLong(col 5:double) -> 23:int, CastTimestampToLong(col 8:timestamp) -> 24:int, CastStringToLong(col 6:string) -> 25:int, CastStringToLong(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 27:int, CastDoubleToLong(col 4:float) -> 28:tinyint, CastDoubleToLo
ng(col 4:float) -> 29:smallint, CastDoubleToLong(col 4:float) -> 30:bigint, CastLongToDouble(col 0:tinyint) -> 31:double, CastLongToDouble(col 1:smallint) -> 32:double, CastLongToDouble(col 2:int) -> 33:double, CastLongToDouble(col 3:bigint) -> 34:double, CastLongToDouble(col 10:boolean) -> 35:double, CastTimestampToDouble(col 8:timestamp) -> 36:double, CastStringToDouble(col 6:string) -> 37:double, CastStringToDouble(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 38:double, CastLongToFloatViaLongToDouble(col 2:int) -> 39:float, CastMillisecondsLongToTimestamp(col 0:tinyint) -> 41:timestamp, CastMillisecondsLongToTimestamp(col 1:smallint) -> 43:timestamp, CastMillisecondsLongToTimestamp(col 2:int) -> 45:timestamp, CastMillisecondsLongToTimestamp(col 3:bigint) -> 47:timestamp, CastDoubleToTimestamp(col 4:float) -> 48:timestamp, CastDoubleToTimestamp(col 5:double) -> 49:timestamp, CastMillisecondsLongToTimestamp(col 10:boolean) -> 51
:timestamp, CastMillisecondsLongToTimestamp(col 52:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 52:bigint) -> 54:timestamp, CastDateToTimestamp(col 52:date)(children: CastTimestampToDate(col 8:timestamp) -> 52:date) -> 55:timestamp, CastStringToTimestamp(col 6:string) -> 56:timestamp, CastStringToTimestamp(col 26:string)(children: StringSubstrColStartLen(col 6:string, start 0, length 1) -> 26:string) -> 57:timestamp, CastLongToString(col 0:tinyint) -> 26:string, CastLongToString(col 1:smallint) -> 58:string, CastLongToString(col 2:int) -> 59:string, CastLongToString(col 3:bigint) -> 60:string, CastFloatToString(col 4:float) -> 61:string, CastDoubleToString(col 5:double) -> 62:string, CastBooleanToStringViaLongToString(col 10:boolean) -> 63:string, CastLongToString(col 52:bigint)(children: LongColMultiplyLongScalar(col 3:bigint, val 0) -> 52:bigint) -> 64:string, CastTimestampToString(col 8:timestamp) -> 65:string, CastStringGroupToChar(col 6:string, maxLength
10) -> 66:char(10), CastStringGroupToVarChar(col 6:string, maxLength 10) -> 67:varchar(10), CastLongToFloatViaLongToDouble(col 52:int)(children: CastDoubleToLong(col 4:float) -> 52:int) -> 68:float, CastLongToDouble(col 52:int)(children: LongColMultiplyLongScalar(col 2:int, val 2) -> 52:int) -> 69:double, CastDoubleToString(col 70:double)(children: FuncSinDoubleToDouble(col 4:float) -> 70:double) -> 71:string, DoubleColAddDoubleColumn(col 70:double, col 72:double)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 70:float, CastLongToDouble(col 10:boolean) -> 72:double) -> 73:double
Statistics: Num rows: 6144 Data size: 16362860 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
@@ -212,7 +212,7 @@ STAGE PLANS:
includeColumns: [0, 1, 2, 3, 4, 5, 6, 8, 10]
dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, double, double, double, double, double, double, double, double, double, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, bigint, timestamp, timestamp, timestamp, timestamp, timestamp, string, string, string, string, string, string, string, string, string, string, string, double, double, double, double, double]
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, string, bigint, bigint, bigint, bigint, double, double, double, double, double, double, double, double, double, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, timestamp, bigint, timestamp, timestamp, timestamp, timestamp, timestamp, string, string, string, string, string, string, string, string, string, string, double, double, double, string, double, double]
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/fa36381f/ql/src/test/results/clientpositive/query_result_fileformat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/query_result_fileformat.q.out b/ql/src/test/results/clientpositive/query_result_fileformat.q.out
index a6d3a20..85030cf 100644
--- a/ql/src/test/results/clientpositive/query_result_fileformat.q.out
+++ b/ql/src/test/results/clientpositive/query_result_fileformat.q.out
@@ -37,12 +37,16 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@nzhang_test1
#### A masked pattern was here ####
1
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization detail
select * from nzhang_test1 where key='key1'
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization detail
select * from nzhang_test1 where key='key1'
POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -55,21 +59,51 @@ STAGE PLANS:
alias: nzhang_test1
filterExpr: (key = 'key1') (type: boolean)
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string, 1:value:string, 2:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterStringGroupColEqualStringScalar(col 0:string, val key1)
predicate: (key = 'key1') (type: boolean)
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 'key1' (type: string), value (type: string)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 1]
+ selectExpressions: ConstantVectorExpression(val key1) -> 3:string
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: key:string, value:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [string]
Stage: Stage-0
Fetch Operator
@@ -110,12 +144,16 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@nzhang_test1
#### A masked pattern was here ####
1
-PREHOOK: query: explain
+PREHOOK: query: explain vectorization detail
select * from nzhang_test1 where key='key1'
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: explain vectorization detail
select * from nzhang_test1 where key='key1'
POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -128,21 +166,51 @@ STAGE PLANS:
alias: nzhang_test1
filterExpr: (key = 'key1') (type: boolean)
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:key:string, 1:value:string, 2:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterStringGroupColEqualStringScalar(col 0:string, val key1)
predicate: (key = 'key1') (type: boolean)
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 'key1' (type: string), value (type: string)
outputColumnNames: _col0, _col1
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [3, 1]
+ selectExpressions: ConstantVectorExpression(val key1) -> 3:string
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 1 Data size: 25 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.row.serde.deserialize IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 2
+ includeColumns: [0, 1]
+ dataColumns: key:string, value:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [string]
Stage: Stage-0
Fetch Operator