You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/10/04 19:37:39 UTC
[3/3] hive git commit: HIVE-20563: Vectorization: CASE WHEN
expression fails when THEN/ELSE type and result type are different (Matt
McCline, reviewed by Teddy Choi)
HIVE-20563: Vectorization: CASE WHEN expression fails when THEN/ELSE type and result type are different (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/97f0513c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/97f0513c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/97f0513c
Branch: refs/heads/master
Commit: 97f0513c4c8ff1c251b1bdd1b84bd238557f03b0
Parents: 857259e
Author: Matt McCline <mm...@hortonworks.com>
Authored: Thu Oct 4 14:37:21 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Thu Oct 4 14:37:21 2018 -0500
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../ql/exec/vector/VectorizationContext.java | 92 ++-
.../expressions/CastTimestampToString.java | 10 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 15 +-
.../exec/vector/TestVectorizationContext.java | 31 +-
.../vector_case_when_conversion.q | 136 ++++
.../llap/vector_case_when_1.q.out | 36 +-
.../llap/vector_case_when_2.q.out | 45 +-
.../llap/vector_case_when_conversion.q.out | 616 +++++++++++++++++++
.../llap/vector_decimal_expressions.q.out | 2 +-
.../llap/vector_udf_adaptor_1.q.out | 52 +-
.../clientpositive/llap/vectorized_case.q.out | 12 +-
.../clientpositive/spark/vectorized_case.q.out | 12 +-
.../clientpositive/vector_case_when_1.q.out | 35 +-
.../clientpositive/vector_case_when_2.q.out | 39 +-
.../vector_decimal_expressions.q.out | 2 +-
.../clientpositive/vectorized_case.q.out | 12 +-
17 files changed, 1061 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fdd8ecc..d444c99 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -763,6 +763,7 @@ minillaplocal.query.files=\
vector_acid4.q,\
vector_annotate_stats_select.q,\
vector_auto_smb_mapjoin_14.q,\
+ vector_case_when_conversion.q,\
vector_char_varchar_1.q,\
vector_complex_all.q,\
vector_complex_join.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 6ca1248..488f277 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -546,6 +546,7 @@ public class VectorizationContext {
private final int initialOutputCol;
private int outputColCount = 0;
private boolean reuseScratchColumns = true;
+ private boolean dontReuseTrackedScratchColumns = false;
protected OutputColumnManager(int initialOutputCol) {
this.initialOutputCol = initialOutputCol;
@@ -558,6 +559,7 @@ public class VectorizationContext {
private String[] scratchVectorTypeNames = new String[100];
private DataTypePhysicalVariation[] scratchDataTypePhysicalVariations =
new DataTypePhysicalVariation[100];
+ private boolean[] scratchColumnTrackWasUsed = new boolean[100];
private final Set<Integer> usedOutputColumns = new HashSet<Integer>();
@@ -589,6 +591,9 @@ public class VectorizationContext {
scratchDataTypePhysicalVariations[i] == dataTypePhysicalVariation)) {
continue;
}
+ if (dontReuseTrackedScratchColumns && scratchColumnTrackWasUsed[i]) {
+ continue;
+ }
//Use i
usedOutputColumns.add(i);
return i;
@@ -597,16 +602,19 @@ public class VectorizationContext {
if (outputColCount < scratchVectorTypeNames.length) {
int newIndex = outputColCount;
scratchVectorTypeNames[outputColCount] = columnType;
- scratchDataTypePhysicalVariations[outputColCount++] = dataTypePhysicalVariation;
+ scratchDataTypePhysicalVariations[outputColCount] = dataTypePhysicalVariation;
+ scratchColumnTrackWasUsed[outputColCount++] = true;
usedOutputColumns.add(newIndex);
return newIndex;
} else {
//Expand the array
scratchVectorTypeNames = Arrays.copyOf(scratchVectorTypeNames, 2*outputColCount);
scratchDataTypePhysicalVariations = Arrays.copyOf(scratchDataTypePhysicalVariations, 2*outputColCount);
+ scratchColumnTrackWasUsed = Arrays.copyOf(scratchColumnTrackWasUsed, 2*outputColCount);
int newIndex = outputColCount;
scratchVectorTypeNames[outputColCount] = columnType;
- scratchDataTypePhysicalVariations[outputColCount++] = dataTypePhysicalVariation;
+ scratchDataTypePhysicalVariations[outputColCount] = dataTypePhysicalVariation;
+ scratchColumnTrackWasUsed[outputColCount++] = true;
usedOutputColumns.add(newIndex);
return newIndex;
}
@@ -647,6 +655,14 @@ public class VectorizationContext {
public void setReuseColumns(boolean reuseColumns) {
this.reuseScratchColumns = reuseColumns;
}
+
+ public void clearScratchColumnWasUsedTracking() {
+ Arrays.fill(scratchColumnTrackWasUsed, false);
+ }
+
+ public void setDontReuseTrackedScratchColumns(boolean dontReuseTrackedScratchColumns) {
+ this.dontReuseTrackedScratchColumns = dontReuseTrackedScratchColumns;
+ }
}
public int allocateScratchColumn(TypeInfo typeInfo) throws HiveException {
@@ -657,6 +673,14 @@ public class VectorizationContext {
return ocm.currentScratchColumns();
}
+ public void clearScratchColumnWasUsedTracking() {
+ ocm.clearScratchColumnWasUsedTracking();
+ }
+
+ public void setDontReuseTrackedScratchColumns(boolean dontReuseTrackedScratchColumns) {
+ ocm.setDontReuseTrackedScratchColumns(dontReuseTrackedScratchColumns);
+ }
+
private VectorExpression getFilterOnBooleanColumnExpression(ExprNodeColumnDesc exprDesc,
int columnNum) throws HiveException {
VectorExpression expr = null;
@@ -1002,6 +1026,33 @@ public class VectorizationContext {
childrenWithCasts.add(child);
}
}
+ } else if (genericUDF instanceof GenericUDFIf) {
+ int i = 0;
+ for (ExprNodeDesc child : children) {
+ TypeInfo castType = commonType;
+ if (i++ == 0) {
+
+ // Skip boolean predicate.
+ childrenWithCasts.add(child);
+ continue;
+ }
+ if (child instanceof ExprNodeConstantDesc &&
+ ((ExprNodeConstantDesc) child).getValue() == null) {
+
+ // Don't cast NULL using ConstantValueExpression but replace the NULL expression with the
+ // desired type set. This lets the doGetIfExpression logic use IfExprCondExprNull, etc.
+ childrenWithCasts.add(new ExprNodeConstantDesc(castType, null));
+ atleastOneCastNeeded = true;
+ continue;
+ }
+ ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, castType);
+ if (castExpression != null) {
+ atleastOneCastNeeded = true;
+ childrenWithCasts.add(castExpression);
+ } else {
+ childrenWithCasts.add(child);
+ }
+ }
} else {
for (ExprNodeDesc child : children) {
ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
@@ -1099,7 +1150,7 @@ public class VectorizationContext {
// Casts to exact types including long to double etc. are needed in some special cases.
if (udf instanceof GenericUDFCoalesce || udf instanceof GenericUDFNvl
- || udf instanceof GenericUDFElt) {
+ || udf instanceof GenericUDFElt || udf instanceof GenericUDFIf) {
GenericUDF genericUdf = getGenericUDFForCast(castType);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
children.add(child);
@@ -2695,8 +2746,11 @@ public class VectorizationContext {
if (null == scalar) {
return null;
}
- PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
+ if (!(type instanceof PrimitiveTypeInfo)) {
+ throw new HiveException("Unsupported type " + typename + " for cast to String");
+ }
+ PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
switch (ptinfo.getPrimitiveCategory()) {
case FLOAT:
case DOUBLE:
@@ -2709,6 +2763,8 @@ public class VectorizationContext {
HiveDecimal decimalVal = (HiveDecimal) scalar;
DecimalTypeInfo decType = (DecimalTypeInfo) type;
return decimalVal.toFormatString(decType.getScale());
+ case TIMESTAMP:
+ return CastTimestampToString.getTimestampString((Timestamp) scalar);
default:
throw new HiveException("Unsupported type "+typename+" for cast to String");
}
@@ -2867,10 +2923,17 @@ public class VectorizationContext {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
- // Return a constant vector expression
- Object constantValue = ((ExprNodeConstantDesc) child).getValue();
- HiveDecimal decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(decimalValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
+
+ // Return a constant vector expression
+ try {
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ HiveDecimal decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(decimalValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
+ } catch (Exception e) {
+
+ // Fall back to VectorUDFAdaptor.
+ return null;
+ }
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
@@ -2907,10 +2970,17 @@ public class VectorizationContext {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
+
// Return a constant vector expression
- Object constantValue = ((ExprNodeConstantDesc) child).getValue();
- String strValue = castConstantToString(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(strValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
+ try {
+ Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+ String strValue = castConstantToString(constantValue, child.getTypeInfo());
+ return getConstantVectorExpression(strValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
+ } catch (Exception e) {
+
+ // Fall back to VectorUDFAdaptor.
+ return null;
+ }
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToString.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToString.java
index 0e20cf1..adc3a9d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToString.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastTimestampToString.java
@@ -31,7 +31,6 @@ import java.time.temporal.ChronoField;
public class CastTimestampToString extends TimestampToStringUnaryUDF {
private static final long serialVersionUID = 1L;
- protected transient Timestamp dt = new Timestamp(0);
private static final DateTimeFormatter PRINT_FORMATTER;
static {
@@ -58,11 +57,16 @@ public class CastTimestampToString extends TimestampToStringUnaryUDF {
@Override
protected void func(BytesColumnVector outV, TimestampColumnVector inV, int i) {
- dt.setTime(inV.time[i]);
- dt.setNanos(inV.nanos[i]);
byte[] temp = LocalDateTime.ofInstant(Instant.ofEpochMilli(inV.time[i]), ZoneOffset.UTC)
.withNano(inV.nanos[i])
.format(PRINT_FORMATTER).getBytes();
assign(outV, i, temp, temp.length);
}
+
+ public static String getTimestampString(Timestamp ts) {
+ return
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(ts.getTime()), ZoneOffset.UTC)
+ .withNano(ts.getNanos())
+ .format(PRINT_FORMATTER);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 51b186c..e93d666 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -4659,6 +4659,14 @@ public class Vectorizer implements PhysicalPlanResolver {
List<ExprNodeDesc> colList = selectDesc.getColList();
int index = 0;
final int size = colList.size();
+
+ // Since the call to fixDecimalDataTypePhysicalVariations will be done post-vector-expression
+ // creation, it cannot freely use deallocated scratch columns. Scratch column reuse assumes
+ // sequential execution so it can reuse freed scratch columns from earlier
+ // evaluations.
+ //
+ vContext.clearScratchColumnWasUsedTracking();
+
VectorExpression[] vectorSelectExprs = new VectorExpression[size];
int[] projectedOutputColumns = new int[size];
for (int i = 0; i < size; i++) {
@@ -4679,7 +4687,12 @@ public class Vectorizer implements PhysicalPlanResolver {
// at least one of its children is DECIMAL_64. Some expressions like x % y for example only accepts DECIMAL
// for x and y (at this time there is only DecimalColModuloDecimalColumn so both x and y has to be DECIMAL).
// The following method introduces a cast if x or y is DECIMAL_64 and parent expression (x % y) is DECIMAL.
- fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs);
+ vContext.setDontReuseTrackedScratchColumns(true);
+ try {
+ fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs);
+ } finally {
+ vContext.setDontReuseTrackedScratchColumns(false);
+ }
vectorSelectDesc.setSelectExpressions(vectorSelectExprs);
vectorSelectDesc.setProjectedOutputColumns(projectedOutputColumns);
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
index 2039ee0..86ac21b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
@@ -38,6 +38,9 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseDoubleTo
import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLogWithBaseLongToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncPowerDoubleToDouble;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprColumnCondExpr;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCondExprColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCondExprCondExpr;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringGroupColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprDoubleColumnDoubleColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprLongColumnLongColumn;
@@ -1269,7 +1272,10 @@ public class TestVectorizationContext {
*/
@Test
public void testIfConditionalExprs() throws HiveException {
- ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Long.class, "col1", "table", false);
+
+ // Predicate.
+ ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Boolean.class, "col1", "table", false);
+
ExprNodeColumnDesc col2Expr = new ExprNodeColumnDesc(Long.class, "col2", "table", false);
ExprNodeColumnDesc col3Expr = new ExprNodeColumnDesc(Long.class, "col3", "table", false);
@@ -1291,6 +1297,7 @@ public class TestVectorizationContext {
columns.add("col2");
columns.add("col3");
VectorizationContext vc = new VectorizationContext("name", columns);
+ exprDesc.setTypeInfo(TypeInfoFactory.longTypeInfo);
VectorExpression ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprLongColumnLongColumn);
@@ -1316,6 +1323,7 @@ public class TestVectorizationContext {
// double column/column IF
children1.set(1, col2Expr);
children1.set(2, col3Expr);
+ exprDesc.setTypeInfo(TypeInfoFactory.doubleTypeInfo);
ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprDoubleColumnDoubleColumn);
@@ -1337,7 +1345,7 @@ public class TestVectorizationContext {
// double scalar/long column IF
children1.set(2, new ExprNodeColumnDesc(Long.class, "col3", "table", false));
ve = vc.getVectorExpression(exprDesc);
- assertTrue(ve instanceof IfExprDoubleScalarLongColumn);
+ assertTrue(ve instanceof IfExprColumnCondExpr);
// Additional combinations of (long,double)X(column,scalar) for each of the second
// and third arguments are omitted. We have coverage of all the source templates
@@ -1350,6 +1358,7 @@ public class TestVectorizationContext {
// timestamp column/column IF
children1.set(1, col2Expr);
children1.set(2, col3Expr);
+ exprDesc.setTypeInfo(TypeInfoFactory.timestampTypeInfo);
ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprTimestampColumnColumn);
@@ -1363,23 +1372,17 @@ public class TestVectorizationContext {
children1.set(2, f);
ve = vc.getVectorExpression(exprDesc);
- // We check for two different classes below because initially the result
- // is IfExprLongColumnLongColumn but in the future if the system is enhanced
- // with constant folding then the result will be IfExprLongColumnLongScalar.
- assertTrue(IfExprTimestampColumnColumn.class == ve.getClass()
- || IfExprTimestampColumnScalar.class == ve.getClass());
+ assertTrue(ve instanceof IfExprTimestampColumnScalar);
// timestamp scalar/scalar
children1.set(1, f);
ve = vc.getVectorExpression(exprDesc);
- assertTrue(IfExprTimestampColumnColumn.class == ve.getClass()
- || IfExprTimestampScalarScalar.class == ve.getClass());
+ assertTrue(ve instanceof IfExprTimestampScalarScalar);
// timestamp scalar/column
children1.set(2, col3Expr);
ve = vc.getVectorExpression(exprDesc);
- assertTrue(IfExprTimestampColumnColumn.class == ve.getClass()
- || IfExprTimestampScalarColumn.class == ve.getClass());
+ assertTrue(ve instanceof IfExprTimestampScalarColumn);
// test for boolean type
col2Expr = new ExprNodeColumnDesc(Boolean.class, "col2", "table", false);
@@ -1388,6 +1391,7 @@ public class TestVectorizationContext {
// column/column
children1.set(1, col2Expr);
children1.set(2, col3Expr);
+ exprDesc.setTypeInfo(TypeInfoFactory.booleanTypeInfo);
ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprLongColumnLongColumn);
@@ -1415,6 +1419,7 @@ public class TestVectorizationContext {
// column/column
children1.set(1, col2Expr);
children1.set(2, col3Expr);
+ exprDesc.setTypeInfo(TypeInfoFactory.stringTypeInfo);
ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
@@ -1444,7 +1449,8 @@ public class TestVectorizationContext {
children1.set(1, col2Expr);
children1.set(2, col3Expr);
ve = vc.getVectorExpression(exprDesc);
- assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
+ exprDesc.setTypeInfo(charTypeInfo);
+ assertTrue(ve instanceof IfExprCondExprCondExpr);
// column/scalar
children1.set(2, constDesc3);
@@ -1471,6 +1477,7 @@ public class TestVectorizationContext {
// column/column
children1.set(1, col2Expr);
children1.set(2, col3Expr);
+ exprDesc.setTypeInfo(varcharTypeInfo);
ve = vc.getVectorExpression(exprDesc);
assertTrue(ve instanceof IfExprStringGroupColumnStringGroupColumn);
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/queries/clientpositive/vector_case_when_conversion.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_case_when_conversion.q b/ql/src/test/queries/clientpositive/vector_case_when_conversion.q
new file mode 100644
index 0000000..04545fd
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_case_when_conversion.q
@@ -0,0 +1,136 @@
+--! qt:dataset:alltypesorc
+set hive.stats.fetch.column.stats=true;
+set hive.explain.user=false;
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+
+-- SORT_QUERY_RESULTS
+
+--
+-- Test "else string"
+--
+EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+-- Force usage of VectorUDFAdaptor
+set hive.test.vectorized.adaptor.override=true;
+
+EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else "none"
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+set hive.test.vectorized.adaptor.override=false;
+
+
+
+-- Test "else null"
+EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+-- Force usage of VectorUDFAdaptor
+set hive.test.vectorized.adaptor.override=true;
+
+EXPLAIN VECTORIZATION ONLY EXPRESSION SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+SELECT cdouble, cstring1, cint, cfloat, csmallint,
+ case
+ when (cdouble is not null) then cdouble
+ when (cstring1 is not null) then cstring1
+ when (cint is not null) then cint
+ when (cfloat is not null) then cfloat
+ when (csmallint is not null) then csmallint
+ else null
+ end as c
+FROM alltypesorc
+WHERE (cdouble IS NULL)
+ORDER BY cdouble, cstring1, cint, cfloat, csmallint, c
+LIMIT 20;
+
+set hive.test.vectorized.adaptor.override=false;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/97f0513c/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
index 4b8544a..c5a35ba 100644
--- a/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_case_when_1.q.out
@@ -205,25 +205,45 @@ STAGE PLANS:
TableScan
alias: lineitem_test
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:l_orderkey:int, 1:l_partkey:int, 2:l_suppkey:int, 3:l_linenumber:int, 4:l_quantity:int, 5:l_extendedprice:double, 6:l_discount:double, 7:l_tax:decimal(10,2)/DECIMAL_64, 8:l_returnflag:char(1), 9:l_linestatus:char(1), 10:l_shipdate:date, 11:l_commitdate:date, 12:l_receiptdate:date, 13:l_shipinstruct:varchar(20), 14:l_shipmode:char(10), 15:l_comment:string, 16:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
expressions: l_quantity (type: int), CASE WHEN ((l_quantity = 1)) THEN ('Single') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN ('Many') ELSE ('Huge number') END (type: string), CASE WHEN ((l_quantity = 1)) THEN ('Single') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN ('Many') ELSE (null) END (type: string), CASE WHEN ((l_quantity = 1)) THEN ('Single') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN (null) ELSE (null) END (type: string), if((l_shipmode = 'SHIP '), date_add(l_shipdate, 10), date_add(l_shipdate, 5)) (type: date), CASE WHEN ((l_returnflag = 'N')) THEN ((l_extendedprice * (1.0D - l_discount))) ELSE (0) END (type: double), CASE WHEN ((l_returnflag = 'N')) THEN ((l_extendedprice * (1.0D - l_discount))) ELSE (0.0D) END (type: double), if((CAST( l_shipinstruct AS STRING)
= 'DELIVER IN PERSON'), null, l_tax) (type: decimal(10,2)), if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, null) (type: decimal(10,2)), if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax) (type: decimal(12,2)), if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0) (type: decimal(12,2)), if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax) (type: decimal(10,2)), if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0) (type: decimal(10,2)), if((l_partkey > 30), CAST( l_receiptdate AS TIMESTAMP), CAST( l_commitdate AS TIMESTAMP)) (type: timestamp), if((l_suppkey > 10000), datediff(l_receiptdate, l_commitdate), null) (type: int), if((l_suppkey > 10000), null, datediff(l_receiptdate, l_commitdate)) (type: int), if(((l_suppkey % 500) > 100), DATE'2009-01-01', DATE'2009-12-31') (type: date)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [4, 21, 22, 23, 20, 24, 25, 27, 28, 29, 30, 31, 32, 35, 37, 38, 19]
+ selectExpressions: VectorUDFAdaptor(CASE WHEN ((l_quantity = 1)) THEN ('Single') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN ('Many') ELSE ('Huge number') END)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string, VectorUDFAdaptor(CASE WHEN ((l_quantity = 1)) THEN ('Single') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN ('Many') ELSE (null) END)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 22:string, VectorUDFAdaptor(CASE WHEN ((l_quantity = 1)) THEN ('Sing
le') WHEN ((l_quantity = 2)) THEN ('Two') WHEN ((l_quantity < 10)) THEN ('Some') WHEN ((l_quantity < 100)) THEN (null) ELSE (null) END)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 23:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 20:date, VectorUDFAdaptor(CASE WHEN ((l_returnflag = 'N')) THEN ((l_extendedprice * (1.0D - l_discount))) ELSE (0) END)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 24:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 24:double) -> 25
:double) -> 24:double, VectorUDFAdaptor(CASE WHEN ((l_returnflag = 'N')) THEN ((l_extendedprice * (1.0D - l_discount))) ELSE (0.0D) END)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 25:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 25:double) -> 26:double) -> 25:double, VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), null, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean) -> 27:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, null))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 17:boolean) -> 28:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string,
val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean) -> 29:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 17:boolean) -> 30:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 17:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(10,2)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean) -> 31:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 18:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 18:boolean) -> 32:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 19:boolean, col 33:timestampcol 34:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30)
-> 19:boolean, CastDateToTimestamp(col 12:date) -> 33:timestamp, CastDateToTimestamp(col 11:date) -> 34:timestamp) -> 35:timestamp, VectorUDFAdaptor(if((l_suppkey > 10000), datediff(l_receiptdate, l_commitdate), null))(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 19:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 36:int) -> 37:int, VectorUDFAdaptor(if((l_suppkey > 10000), null, datediff(l_receiptdate, l_commitdate)))(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 19:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 36:int) -> 38:int, IfExprLongScalarLongScalar(col 36:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 19:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 19:int) -> 36:boolean) -> 19:date
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
enabled: true
enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- notVectorizedReason: SELECT operator: Unexpected hive type name void
- vectorized: false
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 16
+ includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14]
+ dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, string, string, string, double, double, double, decimal(10,2), decimal(10,2), decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, timestamp, timestamp, timestamp, bigint, bigint, bigint]
Stage: Stage-0
Fetch Operator
@@ -531,8 +551,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [4, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 40, 42, 45, 46]
- selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean
, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColSca
lar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnLongScalar(col 17:boolean, col 28:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 29:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 28:double) -> 29:double) -> 28:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 48:decimal(10,2)) -> 30:decimal
(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 19:boolean)
-> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExprNullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:int) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean,
val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date
+ projectedOutputColumnNums: [4, 22, 24, 25, 26, 29, 27, 30, 31, 32, 33, 34, 36, 40, 42, 45, 46]
+ selectExpressions: IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 21:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 22:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 21:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprStringScalarStringScalar(col 20:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean) -> 21:string) -> 22:string) -> 21:string) -> 22:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 24:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean
, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprColumnNull(col 20:boolean, col 21:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 20:boolean, ConstantVectorExpression(val Many) -> 21:string) -> 23:string) -> 24:string) -> 23:string) -> 24:string, IfExprStringScalarStringGroupColumn(col 17:boolean, val Singlecol 23:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, IfExprStringScalarStringGroupColumn(col 18:boolean, val Twocol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 18:boolean, IfExprStringScalarStringGroupColumn(col 19:boolean, val Somecol 23:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 19:boolean, IfExprNullNull(null, null) -> 23:string) -> 25:string) -> 23:string) -> 25:string, IfExprLongColumnLongColumn(col 17:boolean, col 18:date, col 19:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 17:boolean, VectorUDFDateAddColSca
lar(col 10:date, val 10) -> 18:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 19:date) -> 26:date, IfExprDoubleColumnDoubleColumn(col 17:boolean, col 28:doublecol 27:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double, ConstantVectorExpression(val 0.0) -> 27:double) -> 29:double, IfExprDoubleColumnDoubleScalar(col 17:boolean, col 28:double, val 0.0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 17:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 27:double) -> 28:double) -> 27:double, IfExprNullColumn(col 17:boolean, null, col 48)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 17:boolean, ConvertDecimal64ToDecimal(col 7:deci
mal(10,2)/DECIMAL_64) -> 48:decimal(10,2)) -> 30:decimal(10,2), IfExprColumnNull(col 18:boolean, col 49:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 18:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 49:decimal(10,2)) -> 31:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 19:boolean) -> 32:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 19:boolean) -> 33:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 19:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(10,2)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVE
R IN PERSON)(children: col 13:varchar(20)) -> 19:boolean) -> 34:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 35:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 35:boolean) -> 36:decimal(10,2)/DECIMAL_64, IfExprTimestampColumnColumn(col 37:boolean, col 38:timestampcol 39:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 37:boolean, CastDateToTimestamp(col 12:date) -> 38:timestamp, CastDateToTimestamp(col 11:date) -> 39:timestamp) -> 40:timestamp, IfExprColumnNull(col 37:boolean, col 41:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 37:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 41:int) -> 42:int, IfExprNullColumn(col 43:boolean, null, col 44)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 43:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 44:in
t) -> 45:int, IfExprLongScalarLongScalar(col 47:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 46:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 46:int) -> 47:boolean) -> 46:date
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -868,8 +888,8 @@ STAGE PLANS:
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [4, 27, 38, 48, 52, 54, 60, 62, 64, 66, 67, 68, 70, 74, 77, 80, 81]
- selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolea
n, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLes
sLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprDoubleColumnLongScalar(col 57:boolean, col 58:double, val 0)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 58:double) -> 54:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 58:double)(children: DoubleScalarSubtractD
oubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 61:boolean, null, col 83)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 61:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 83:decimal(10,2)) -> 62:decimal(10,2), IfExprColumnNull(col 63:boolean, col 84:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 63:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 84:decimal(10,2)) -> 64:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 65:boolean) -> 66:decimal(12,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'TAKE BACK RETURN'), l_tax
, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 65:boolean) -> 67:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 65:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(1,0)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 65:boolean) -> 68:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 69:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 69:boolean) -> 70:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 71:boolean, col 72:timestampcol 73:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 71:boolean, CastDateToTimestamp(col 12:date) -> 72:timestamp, CastDateToTimestamp(col 11:date) -> 73:timestamp) -> 74:timestamp, IfExprCondExprNull(col 75:boolean, col 76:int
, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 75:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 76:int) -> 77:int, IfExprNullCondExpr(col 78:boolean, null, col 79:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 78:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 79:int) -> 80:int, IfExprLongScalarLongScalar(col 82:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 81:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 81:int) -> 82:boolean) -> 81:date
+ projectedOutputColumnNums: [4, 27, 38, 48, 52, 56, 60, 62, 64, 69, 73, 74, 76, 80, 83, 86, 87]
+ selectExpressions: IfExprColumnCondExpr(col 17:boolean, col 18:stringcol 26:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 17:boolean, ConstantVectorExpression(val Single) -> 18:string, IfExprColumnCondExpr(col 19:boolean, col 20:stringcol 25:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 19:boolean, ConstantVectorExpression(val Two) -> 20:string, IfExprColumnCondExpr(col 21:boolean, col 22:stringcol 24:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 21:boolean, ConstantVectorExpression(val Some) -> 22:string, IfExprStringScalarStringScalar(col 23:boolean, val Many, val Huge number)(children: LongColLessLongScalar(col 4:int, val 100) -> 23:boolean) -> 24:string) -> 25:string) -> 26:string) -> 27:string, IfExprColumnCondExpr(col 23:boolean, col 28:stringcol 37:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 23:boolean, ConstantVectorExpression(val Single) -> 28:string, IfExprColumnCondExpr(col 29:boolea
n, col 30:stringcol 36:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 29:boolean, ConstantVectorExpression(val Two) -> 30:string, IfExprColumnCondExpr(col 31:boolean, col 32:stringcol 35:string)(children: LongColLessLongScalar(col 4:int, val 10) -> 31:boolean, ConstantVectorExpression(val Some) -> 32:string, IfExprColumnNull(col 33:boolean, col 34:string, null)(children: LongColLessLongScalar(col 4:int, val 100) -> 33:boolean, ConstantVectorExpression(val Many) -> 34:string) -> 35:string) -> 36:string) -> 37:string) -> 38:string, IfExprColumnCondExpr(col 39:boolean, col 40:stringcol 47:string)(children: LongColEqualLongScalar(col 4:int, val 1) -> 39:boolean, ConstantVectorExpression(val Single) -> 40:string, IfExprColumnCondExpr(col 41:boolean, col 42:stringcol 46:string)(children: LongColEqualLongScalar(col 4:int, val 2) -> 41:boolean, ConstantVectorExpression(val Two) -> 42:string, IfExprColumnCondExpr(col 43:boolean, col 44:stringcol 45:string)(children: LongColLes
sLongScalar(col 4:int, val 10) -> 43:boolean, ConstantVectorExpression(val Some) -> 44:string, IfExprNullNull(null, null) -> 45:string) -> 46:string) -> 47:string) -> 48:string, IfExprCondExprCondExpr(col 49:boolean, col 50:datecol 51:date)(children: StringGroupColEqualCharScalar(col 14:char(10), val SHIP) -> 49:boolean, VectorUDFDateAddColScalar(col 10:date, val 10) -> 50:date, VectorUDFDateAddColScalar(col 10:date, val 5) -> 51:date) -> 52:date, IfExprCondExprCondExpr(col 53:boolean, col 55:doublecol 54:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 53:boolean, DoubleColMultiplyDoubleColumn(col 5:double, col 54:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 54:double) -> 55:double, ConstantVectorExpression(val 0.0) -> 54:double) -> 56:double, IfExprCondExprColumn(col 57:boolean, col 59:double, col 58:double)(children: StringGroupColEqualCharScalar(col 8:char(1), val N) -> 57:boolean, DoubleColMultiplyDoubleColumn(col 5:double
, col 58:double)(children: DoubleScalarSubtractDoubleColumn(val 1.0, col 6:double) -> 58:double) -> 59:double, ConstantVectorExpression(val 0.0) -> 58:double) -> 60:double, IfExprNullColumn(col 61:boolean, null, col 89)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 61:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 89:decimal(10,2)) -> 62:decimal(10,2), IfExprColumnNull(col 63:boolean, col 90:decimal(10,2), null)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 63:boolean, ConvertDecimal64ToDecimal(col 7:decimal(10,2)/DECIMAL_64) -> 90:decimal(10,2)) -> 64:decimal(10,2), VectorUDFAdaptor(if((CAST( l_shipinstruct AS STRING) = 'DELIVER IN PERSON'), 0, l_tax))(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 68:boolean) -> 69:decimal(12,2), VectorUDFAdaptor(if((CAST( l_ship
instruct AS STRING) = 'TAKE BACK RETURN'), l_tax, 0))(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 72:boolean) -> 73:decimal(12,2), IfExprDecimal64ScalarDecimal64Column(col 72:boolean, decimal64Val 0, decimalVal 0, col 7:decimal(10,2)/DECIMAL_64)(children: StringGroupColEqualStringScalar(col 13:string, val DELIVER IN PERSON)(children: col 13:varchar(20)) -> 72:boolean) -> 74:decimal(10,2)/DECIMAL_64, IfExprDecimal64ColumnDecimal64Scalar(col 75:boolean, col 7:decimal(10,2)/DECIMAL_64, decimal64Val 0, decimalVal 0)(children: StringGroupColEqualStringScalar(col 13:string, val TAKE BACK RETURN)(children: col 13:varchar(20)) -> 75:boolean) -> 76:decimal(10,2)/DECIMAL_64, IfExprCondExprCondExpr(col 77:boolean, col 78:timestampcol 79:timestamp)(children: LongColGreaterLongScalar(col 1:int, val 30) -> 77:boolean, CastDateToTimestamp(col 12:date) -> 78:timestamp, CastDateToTimestamp(col 11:date) -> 79:timestamp) -> 80:timesta
mp, IfExprCondExprNull(col 81:boolean, col 82:int, null)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 81:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 82:int) -> 83:int, IfExprNullCondExpr(col 84:boolean, null, col 85:int)(children: LongColGreaterLongScalar(col 2:int, val 10000) -> 84:boolean, VectorUDFDateDiffColCol(col 12:date, col 11:date) -> 85:int) -> 86:int, IfExprLongScalarLongScalar(col 88:boolean, val 14245, val 14609)(children: LongColGreaterLongScalar(col 87:int, val 100)(children: LongColModuloLongScalar(col 2:int, val 500) -> 87:int) -> 88:boolean) -> 87:date
Statistics: Num rows: 101 Data size: 57327 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -897,7 +917,7 @@ STAGE PLANS:
includeColumns: [1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14]
dataColumns: l_orderkey:int, l_partkey:int, l_suppkey:int, l_linenumber:int, l_quantity:int, l_extendedprice:double, l_discount:double, l_tax:decimal(10,2)/DECIMAL_64, l_returnflag:char(1), l_linestatus:char(1), l_shipdate:date, l_commitdate:date, l_receiptdate:date, l_shipinstruct:varchar(20), l_shipmode:char(10), l_comment:string
partitionColumnCount: 0
- scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, bigint, bigint, double, double, double, bigint, decimal(10,2), bigint, decimal(10,2), bigint, decimal(12,2), decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)]
+ scratchColumnTypeNames: [bigint, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, string, bigint, string, bigint, string, string, string, string, string, bigint, bigint, bigint, bigint, bigint, double, double, double, bigint, double, double, double, bigint, decimal(10,2), bigint, decimal(10,2), bigint, bigint, decimal(12,2), bigint, decimal(12,2), decimal(12,2), bigint, bigint, decimal(12,2), decimal(10,2)/DECIMAL_64, bigint, decimal(10,2)/DECIMAL_64, bigint, timestamp, timestamp, timestamp, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, decimal(10,2), decimal(10,2)]
Stage: Stage-0
Fetch Operator