You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2017/11/29 18:01:32 UTC
[18/29] hive git commit: HIVE-17528 : Add more q-tests for
Hive-on-Spark with Parquet vectorized reader (Ferdinand Xu,
reviewed by Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/9a59592e/ql/src/test/results/clientpositive/parquet_vectorization_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_7.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_7.q.out
new file mode 100644
index 0000000..77caa03
--- /dev/null
+++ b/ql/src/test/results/clientpositive/parquet_vectorization_7.q.out
@@ -0,0 +1,435 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesparquet
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterExprOrExpr(children: FilterDoubleScalarLessDoubleColumn(val 988888.0, col 5:double), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -15.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDoubleScalarGreaterEqualDoubleColumn(val 3569.0, col 5:double))))
+ predicate: (((988888.0 < cdouble) or ((UDFToDouble(ctimestamp2) > -15.0) and (3569.0 >= cdouble))) and ((UDFToDouble(ctimestamp1) <= 0.0) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0)) (type: boolean)
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 14, 15, 16, 17, 19, 20, 18, 21, 23]
+ selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 14:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 15:int, LongColUnaryMinus(col 1:smallint) -> 16:smallint, LongColUnaryMinus(col 0:tinyint) -> 17:tinyint, LongColAddLongScalar(col 18:int, val 17)(children: col 18:tinyint) -> 19:int, LongColMultiplyLongColumn(col 3:bigint, col 18:bigint)(children: col 18:smallint) -> 20:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 18:int, LongColUnaryMinus(col 0:tinyint) -> 21:tinyint, LongColModuloLongColumn(col 22:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 22:tinyint) -> 23:tinyint
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint)
+ sort order: +++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [0, 1, 2, 3, 5, 6, 7, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint]
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: smallint), KEY.reducesinkkey3 (type: tinyint), KEY.reducesinkkey4 (type: timestamp), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: bigint), KEY.reducesinkkey7 (type: int), KEY.reducesinkkey8 (type: smallint), KEY.reducesinkkey9 (type: tinyint), KEY.reducesinkkey10 (type: int), KEY.reducesinkkey11 (type: bigint), KEY.reducesinkkey12 (type: int), KEY.reducesinkkey9 (type: tinyint), KEY.reducesinkkey14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 25
+ Statistics: Num rows: 25 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 25 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 25
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > -15)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+NULL -2118149242 -7196 56 1969-12-31 15:59:50.462 NULL -4236298484 0 7196 -56 -39 -15242201945432 NULL -56 0
+NULL -2121399625 -7196 27 1969-12-31 15:59:50.046 NULL -4242799250 0 7196 -27 -10 -15265591701500 NULL -27 0
+NULL -2124802690 -7196 -6 1969-12-31 15:59:57.92 NULL -4249605380 0 7196 6 23 -15290080157240 NULL 6 0
+NULL -2128720310 -7196 -52 1969-12-31 15:59:45.978 NULL -4257440620 0 7196 52 69 -15318271350760 NULL 52 0
+NULL -2132232110 -200 60 1969-12-31 15:59:47.019 NULL -4264464220 -200 200 -60 -43 -426446422000 NULL -60 0
+NULL -2132536965 -7196 9 1969-12-31 15:59:46 NULL -4265073930 0 7196 -9 8 -15345736000140 NULL -9 0
+NULL -2135141157 -7196 50 1969-12-31 15:59:50.192 NULL -4270282314 0 7196 -50 -33 -15364475765772 NULL -50 0
+NULL -2137537679 -7196 -25 1969-12-31 15:59:50.136 NULL -4275075358 0 7196 25 42 -15381721138084 NULL 25 0
+NULL -2145481991 -7196 56 1969-12-31 15:59:55.667 NULL -4290963982 0 7196 -56 -39 -15438888407236 NULL -56 0
+NULL NULL -200 -36 1969-12-31 15:59:57.241 NULL NULL -200 200 36 53 NULL NULL 36 0
+NULL NULL -200 -43 1969-12-31 15:59:53.783 NULL NULL -200 200 43 60 NULL NULL 43 0
+NULL NULL -200 -58 1969-12-31 15:59:51.115 NULL NULL -200 200 58 75 NULL NULL 58 0
+NULL NULL -200 22 1969-12-31 15:59:50.109 NULL NULL -200 200 -22 -5 NULL NULL -22 0
+NULL NULL -200 3 1969-12-31 15:59:50.489 NULL NULL -200 200 -3 14 NULL NULL -3 0
+NULL NULL -200 43 1969-12-31 15:59:57.003 NULL NULL -200 200 -43 -26 NULL NULL -43 0
+NULL NULL -200 53 1969-12-31 15:59:49.46 NULL NULL -200 200 -53 -36 NULL NULL -53 0
+NULL NULL -200 9 1969-12-31 15:59:44.108 NULL NULL -200 200 -9 8 NULL NULL -9 0
+NULL NULL -7196 -38 1969-12-31 15:59:53.503 NULL NULL 0 7196 38 55 NULL NULL 38 0
+NULL NULL -7196 -49 1969-12-31 15:59:51.009 NULL NULL 0 7196 49 66 NULL NULL 49 0
+NULL NULL -7196 -49 1969-12-31 15:59:52.052 NULL NULL 0 7196 49 66 NULL NULL 49 0
+NULL NULL -7196 -50 1969-12-31 15:59:52.424 NULL NULL 0 7196 50 67 NULL NULL 50 0
+NULL NULL -7196 -61 1969-12-31 15:59:44.823 NULL NULL 0 7196 61 78 NULL NULL 61 0
+NULL NULL -7196 1 1969-12-31 15:59:48.361 NULL NULL 0 7196 -1 16 NULL NULL -1 0
+NULL NULL -7196 14 1969-12-31 15:59:50.291 NULL NULL 0 7196 -14 3 NULL NULL -14 0
+NULL NULL -7196 22 1969-12-31 15:59:52.699 NULL NULL 0 7196 -22 -5 NULL NULL -22 0
+PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesparquet
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 0:tinyint, val 0), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 13:double, val 0.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterExprOrExpr(children: FilterDoubleScalarLessDoubleColumn(val 988888.0, col 5:double), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val 7.6850000000000005)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDoubleScalarGreaterEqualDoubleColumn(val 3569.0, col 5:double))))
+ predicate: (((988888.0 < cdouble) or ((UDFToDouble(ctimestamp2) > 7.6850000000000005) and (3569.0 >= cdouble))) and ((UDFToDouble(ctimestamp1) <= 0.0) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0)) (type: boolean)
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 14, 15, 16, 17, 19, 20, 18, 21, 23]
+ selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 14:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 15:int, LongColUnaryMinus(col 1:smallint) -> 16:smallint, LongColUnaryMinus(col 0:tinyint) -> 17:tinyint, LongColAddLongScalar(col 18:int, val 17)(children: col 18:tinyint) -> 19:int, LongColMultiplyLongColumn(col 3:bigint, col 18:bigint)(children: col 18:smallint) -> 20:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 18:int, LongColUnaryMinus(col 0:tinyint) -> 21:tinyint, LongColModuloLongColumn(col 22:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 22:tinyint) -> 23:tinyint
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint)
+ sort order: +++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: bigint), KEY.reducesinkkey2 (type: smallint), KEY.reducesinkkey3 (type: tinyint), KEY.reducesinkkey4 (type: timestamp), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: bigint), KEY.reducesinkkey7 (type: int), KEY.reducesinkkey8 (type: smallint), KEY.reducesinkkey9 (type: tinyint), KEY.reducesinkkey10 (type: int), KEY.reducesinkkey11 (type: bigint), KEY.reducesinkkey12 (type: int), KEY.reducesinkkey9 (type: tinyint), KEY.reducesinkkey14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 5461 Data size: 65532 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 25
+ Statistics: Num rows: 25 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 25 Data size: 300 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 25
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cboolean1,
+ cbigint,
+ csmallint,
+ ctinyint,
+ ctimestamp1,
+ cstring1,
+ (cbigint + cbigint) as c1,
+ (csmallint % -257) as c2,
+ (-(csmallint)) as c3,
+ (-(ctinyint)) as c4,
+ ((-(ctinyint)) + 17) as c5,
+ (cbigint * (-(csmallint))) as c6,
+ (cint % csmallint) as c7,
+ (-(ctinyint)) as c8,
+ ((-(ctinyint)) % ctinyint) as c9
+FROM alltypesparquet
+WHERE ((ctinyint != 0)
+ AND (((ctimestamp1 <= 0.0)
+ OR ((ctinyint = cint)
+ OR (cstring2 LIKE 'ss')))
+ AND ((988888 < cdouble)
+ OR ((ctimestamp2 > 7.6850000000000005)
+ AND (3569 >= cdouble)))))
+ORDER BY cboolean1, cbigint, csmallint, ctinyint, ctimestamp1, cstring1, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 25
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+true NULL -14584 -7 1969-12-31 15:59:46.965 1cGVWH7n1QU NULL -192 14584 7 24 NULL 5864 7 0
+true NULL -14739 4 1969-12-31 15:59:55.188 cvLH6Eat2yFsyy7p NULL -90 14739 -4 13 NULL 8966 -4 0
+true NULL -14771 13 1969-12-31 15:59:58.839 821UdmGbkEf4j NULL -122 14771 -13 4 NULL 12897 -13 0
+true NULL -15149 20 1969-12-31 15:59:46.575 1cGVWH7n1QU NULL -243 15149 -20 -3 NULL 10520 -20 0
+true NULL -15344 48 1969-12-31 15:59:54.853 821UdmGbkEf4j NULL -181 15344 -48 -31 NULL 13871 -48 0
+true NULL -15388 49 1969-12-31 15:59:58.545 cvLH6Eat2yFsyy7p NULL -225 15388 -49 -32 NULL 3131 -49 0
+true NULL -15431 -11 1969-12-31 15:59:52.176 cvLH6Eat2yFsyy7p NULL -11 15431 11 28 NULL 7586 11 0
+true NULL -15435 -21 1969-12-31 15:59:54.705 1cGVWH7n1QU NULL -15 15435 21 38 NULL 6586 21 0
+true NULL -15450 -34 1969-12-31 15:59:55.681 cvLH6Eat2yFsyy7p NULL -30 15450 34 51 NULL 5717 34 0
+true NULL -15478 7 1969-12-31 15:59:56.809 cvLH6Eat2yFsyy7p NULL -58 15478 -7 10 NULL 7501 -7 0
+true NULL -15555 -2 1969-12-31 15:59:58.773 1cGVWH7n1QU NULL -135 15555 2 19 NULL 9991 2 0
+true NULL -15659 -11 1969-12-31 15:59:48.466 cvLH6Eat2yFsyy7p NULL -239 15659 11 28 NULL 12199 11 0
+true NULL -15711 -60 1969-12-31 15:59:53.115 821UdmGbkEf4j NULL -34 15711 60 77 NULL 7646 60 0
+true NULL -15770 -23 1969-12-31 15:59:56.822 1cGVWH7n1QU NULL -93 15770 23 40 NULL 4926 23 0
+true NULL -15794 53 1969-12-31 15:59:53.749 cvLH6Eat2yFsyy7p NULL -117 15794 -53 -36 NULL 4351 -53 0
+true NULL -15892 29 1969-12-31 15:59:57.937 821UdmGbkEf4j NULL -215 15892 -29 -12 NULL 171 -29 0
+true NULL -15899 50 1969-12-31 15:59:46.926 821UdmGbkEf4j NULL -222 15899 -50 -33 NULL 10210 -50 0
+true NULL -15920 -64 1969-12-31 15:59:51.859 cvLH6Eat2yFsyy7p NULL -243 15920 64 81 NULL 6687 64 0
+true NULL -15922 -17 1969-12-31 15:59:46.164 821UdmGbkEf4j NULL -245 15922 17 34 NULL 10851 17 0
+true NULL -15980 -6 1969-12-31 15:59:54.84 1cGVWH7n1QU NULL -46 15980 6 23 NULL 14836 6 0
+true NULL -16017 -21 1969-12-31 15:59:44.02 821UdmGbkEf4j NULL -83 16017 21 38 NULL 2282 21 0
+true NULL -16036 -15 1969-12-31 15:59:58.681 1cGVWH7n1QU NULL -102 16036 15 32 NULL 7928 15 0
+true NULL -16076 59 1969-12-31 15:59:55.023 821UdmGbkEf4j NULL -142 16076 -59 -42 NULL 7907 -59 0
+true NULL -16122 50 1969-12-31 15:59:51.608 1cGVWH7n1QU NULL -188 16122 -50 -33 NULL 1828 -50 0
+true NULL -16153 35 1969-12-31 15:59:52.036 1cGVWH7n1QU NULL -219 16153 -35 -18 NULL 14817 -35 0
http://git-wip-us.apache.org/repos/asf/hive/blob/9a59592e/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out b/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out
new file mode 100644
index 0000000..caa4bff
--- /dev/null
+++ b/ql/src/test/results/clientpositive/parquet_vectorization_8.q.out
@@ -0,0 +1,409 @@
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesparquet
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct<transactionid:bigint,bucketid:int,rowid:bigint>]
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0)))
+ predicate: ((cboolean1 is not null and (cdouble = 988888.0)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 10.0) and (UDFToDouble(ctimestamp2) <> 16.0))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15 - cdouble) (type: double), (cdouble * -257.0) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15 - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [8, 5, 10, 6, 4, 13, 14, 15, 17, 19, 16, 18, 20, 22]
+ selectExpressions: DoubleColUnaryMinus(col 5:double) -> 13:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 14:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 15:double, DoubleColAddDoubleColumn(col 16:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 16:float) -> 17:float, DoubleColAddDoubleColumn(col 16:double, col 18:double)(children: DoubleColUnaryMinus(col 5:double) -> 16:double, CastLongToDouble(col 3:bigint) -> 18:double) -> 19:double, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 18:float, DoubleColUnaryMinus(col 4:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 23:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 21:double, col 23:float) -> 22:double
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double)
+ sort order: ++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 12
+ includeColumns: [2, 3, 4, 5, 6, 7, 8, 9, 10]
+ dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [double, double, double, double, double, double, double, double, double, double, double]
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: boolean), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: float), KEY.reducesinkkey5 (type: double), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double), KEY.reducesinkkey8 (type: float), KEY.reducesinkkey9 (type: double), KEY.reducesinkkey5 (type: double), KEY.reducesinkkey11 (type: float), KEY.reducesinkkey12 (type: float), KEY.reducesinkkey13 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 10)
+ AND (ctimestamp2 != 16)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+1969-12-31 15:59:30.929 -200.0 NULL NULL 12.0 200.0 -5438.15 51400.0 NULL 2.128824065E9 200.0 -13.389 -12.0 NULL
+1969-12-31 15:59:30.929 -200.0 NULL NULL 42.0 200.0 -5438.15 51400.0 NULL -5.78935585E8 200.0 -43.389 -42.0 NULL
+1969-12-31 15:59:30.929 -7196.0 NULL NULL 49.0 7196.0 1557.8500000000004 1849372.0 NULL 1.948230809E9 7196.0 -50.389 -49.0 NULL
+1969-12-31 15:59:30.929 15601.0 NULL NULL -54.0 -15601.0 -21239.15 -4009457.0 NULL -1.114185408E9 -15601.0 52.611 54.0 NULL
+1969-12-31 15:59:43.628 -200.0 NULL NULL 23.0 200.0 -5438.15 51400.0 NULL 2.034056161E9 200.0 -24.389 -23.0 NULL
+1969-12-31 15:59:43.637 -7196.0 NULL NULL -44.0 7196.0 1557.8500000000004 1849372.0 NULL 1.100301451E9 7196.0 42.611 44.0 NULL
+1969-12-31 15:59:43.64 -7196.0 NULL NULL -51.0 7196.0 1557.8500000000004 1849372.0 NULL -1.339157623E9 7196.0 49.611 51.0 NULL
+1969-12-31 15:59:43.661 -200.0 NULL NULL 13.0 200.0 -5438.15 51400.0 NULL -6.8063459E7 200.0 -14.389 -13.0 NULL
+1969-12-31 15:59:43.676 -200.0 NULL NULL -10.0 200.0 -5438.15 51400.0 NULL 8.34296111E8 200.0 8.611 10.0 NULL
+1969-12-31 15:59:43.705 15601.0 NULL NULL -12.0 -15601.0 -21239.15 -4009457.0 NULL 5.95398978E8 -15601.0 10.611 12.0 NULL
+1969-12-31 15:59:43.709 15601.0 NULL NULL 61.0 -15601.0 -21239.15 -4009457.0 NULL -1.55652257E8 -15601.0 -62.389 -61.0 NULL
+1969-12-31 15:59:43.72 -200.0 NULL NULL -18.0 200.0 -5438.15 51400.0 NULL 1.809037075E9 200.0 16.611 18.0 NULL
+1969-12-31 15:59:43.721 15601.0 NULL NULL 57.0 -15601.0 -21239.15 -4009457.0 NULL -1.465672807E9 -15601.0 -58.389 -57.0 NULL
+1969-12-31 15:59:43.749 -7196.0 NULL NULL -39.0 7196.0 1557.8500000000004 1849372.0 NULL 3.96436076E8 7196.0 37.611 39.0 NULL
+1969-12-31 15:59:43.771 15601.0 NULL NULL 49.0 -15601.0 -21239.15 -4009457.0 NULL -1.10733693E9 -15601.0 -50.389 -49.0 NULL
+1969-12-31 15:59:43.773 -200.0 NULL NULL -24.0 200.0 -5438.15 51400.0 NULL -1.046049502E9 200.0 22.611 24.0 NULL
+1969-12-31 15:59:43.782 -7196.0 NULL NULL 51.0 7196.0 1557.8500000000004 1849372.0 NULL -1.607339819E9 7196.0 -52.389 -51.0 NULL
+1969-12-31 15:59:43.783 -200.0 NULL NULL -11.0 200.0 -5438.15 51400.0 NULL 1.2116287E7 200.0 9.611 11.0 NULL
+1969-12-31 15:59:43.807 -7196.0 NULL NULL 42.0 7196.0 1557.8500000000004 1849372.0 NULL -5.98226333E8 7196.0 -43.389 -42.0 NULL
+1969-12-31 15:59:43.82 -7196.0 NULL NULL -30.0 7196.0 1557.8500000000004 1849372.0 NULL 1.329550715E9 7196.0 28.611 30.0 NULL
+PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION
+SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypesparquet
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TableScan Vectorization:
+ native: true
+ Filter Operator
+ Filter Vectorization:
+ className: VectorFilterOperator
+ native: true
+ predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: SelectColumnIsNotNull(col 7:string), FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 13:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double)), FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0)))
+ predicate: ((cboolean1 is not null and (cdouble = 988888.0)) or (cfloat < -6432) or (cstring2 is not null and (UDFToDouble(ctimestamp1) <= 12.503) and (UDFToDouble(ctimestamp2) <> 11.998))) (type: boolean)
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15 - cdouble) (type: double), (cdouble * -257.0) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15 - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [8, 5, 10, 6, 4, 13, 14, 15, 17, 19, 16, 18, 20, 22]
+ selectExpressions: DoubleColUnaryMinus(col 5:double) -> 13:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 14:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 15:double, DoubleColAddDoubleColumn(col 16:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 16:float) -> 17:float, DoubleColAddDoubleColumn(col 16:double, col 18:double)(children: DoubleColUnaryMinus(col 5:double) -> 16:double, CastLongToDouble(col 3:bigint) -> 18:double) -> 19:double, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 18:float, DoubleColUnaryMinus(col 4:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 23:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 21:double, col 23:float) -> 22:double
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double)
+ sort order: ++++++++++++++
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: []
+ featureSupportInUse: []
+ inputFileFormats: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: timestamp), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: boolean), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: float), KEY.reducesinkkey5 (type: double), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey7 (type: double), KEY.reducesinkkey8 (type: float), KEY.reducesinkkey9 (type: double), KEY.reducesinkkey5 (type: double), KEY.reducesinkkey11 (type: float), KEY.reducesinkkey12 (type: float), KEY.reducesinkkey13 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 12288 Data size: 147456 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 20 Data size: 240 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 20
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT ctimestamp1,
+ cdouble,
+ cboolean1,
+ cstring1,
+ cfloat,
+ (-(cdouble)) as c1,
+ (-5638.15 - cdouble) as c2,
+ (cdouble * -257) as c3,
+ (cint + cfloat) as c4,
+ ((-(cdouble)) + cbigint) as c5,
+ (-(cdouble)) as c6,
+ (-1.389 - cfloat) as c7,
+ (-(cfloat)) as c8,
+ ((-5638.15 - cdouble) + (cint + cfloat)) as c9
+FROM alltypesparquet
+WHERE (((cstring2 IS NOT NULL)
+ AND ((ctimestamp1 <= 12.503)
+ AND (ctimestamp2 != 11.998)))
+ OR ((cfloat < -6432)
+ OR ((cboolean1 IS NOT NULL)
+ AND (cdouble = 988888))))
+ORDER BY ctimestamp1, cdouble, cboolean1, cstring1, cfloat, c1, c2, c3, c4, c5, c6, c7, c8, c9
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesparquet
+#### A masked pattern was here ####
+1969-12-31 15:59:30.929 -200.0 NULL NULL 12.0 200.0 -5438.15 51400.0 NULL 2.128824065E9 200.0 -13.389 -12.0 NULL
+1969-12-31 15:59:30.929 -200.0 NULL NULL 42.0 200.0 -5438.15 51400.0 NULL -5.78935585E8 200.0 -43.389 -42.0 NULL
+1969-12-31 15:59:30.929 -7196.0 NULL NULL 49.0 7196.0 1557.8500000000004 1849372.0 NULL 1.948230809E9 7196.0 -50.389 -49.0 NULL
+1969-12-31 15:59:30.929 15601.0 NULL NULL -54.0 -15601.0 -21239.15 -4009457.0 NULL -1.114185408E9 -15601.0 52.611 54.0 NULL
+1969-12-31 15:59:43.628 -200.0 NULL NULL 23.0 200.0 -5438.15 51400.0 NULL 2.034056161E9 200.0 -24.389 -23.0 NULL
+1969-12-31 15:59:43.637 -7196.0 NULL NULL -44.0 7196.0 1557.8500000000004 1849372.0 NULL 1.100301451E9 7196.0 42.611 44.0 NULL
+1969-12-31 15:59:43.64 -7196.0 NULL NULL -51.0 7196.0 1557.8500000000004 1849372.0 NULL -1.339157623E9 7196.0 49.611 51.0 NULL
+1969-12-31 15:59:43.661 -200.0 NULL NULL 13.0 200.0 -5438.15 51400.0 NULL -6.8063459E7 200.0 -14.389 -13.0 NULL
+1969-12-31 15:59:43.676 -200.0 NULL NULL -10.0 200.0 -5438.15 51400.0 NULL 8.34296111E8 200.0 8.611 10.0 NULL
+1969-12-31 15:59:43.705 15601.0 NULL NULL -12.0 -15601.0 -21239.15 -4009457.0 NULL 5.95398978E8 -15601.0 10.611 12.0 NULL
+1969-12-31 15:59:43.709 15601.0 NULL NULL 61.0 -15601.0 -21239.15 -4009457.0 NULL -1.55652257E8 -15601.0 -62.389 -61.0 NULL
+1969-12-31 15:59:43.72 -200.0 NULL NULL -18.0 200.0 -5438.15 51400.0 NULL 1.809037075E9 200.0 16.611 18.0 NULL
+1969-12-31 15:59:43.721 15601.0 NULL NULL 57.0 -15601.0 -21239.15 -4009457.0 NULL -1.465672807E9 -15601.0 -58.389 -57.0 NULL
+1969-12-31 15:59:43.749 -7196.0 NULL NULL -39.0 7196.0 1557.8500000000004 1849372.0 NULL 3.96436076E8 7196.0 37.611 39.0 NULL
+1969-12-31 15:59:43.771 15601.0 NULL NULL 49.0 -15601.0 -21239.15 -4009457.0 NULL -1.10733693E9 -15601.0 -50.389 -49.0 NULL
+1969-12-31 15:59:43.773 -200.0 NULL NULL -24.0 200.0 -5438.15 51400.0 NULL -1.046049502E9 200.0 22.611 24.0 NULL
+1969-12-31 15:59:43.782 -7196.0 NULL NULL 51.0 7196.0 1557.8500000000004 1849372.0 NULL -1.607339819E9 7196.0 -52.389 -51.0 NULL
+1969-12-31 15:59:43.783 -200.0 NULL NULL -11.0 200.0 -5438.15 51400.0 NULL 1.2116287E7 200.0 9.611 11.0 NULL
+1969-12-31 15:59:43.807 -7196.0 NULL NULL 42.0 7196.0 1557.8500000000004 1849372.0 NULL -5.98226333E8 7196.0 -43.389 -42.0 NULL
+1969-12-31 15:59:43.82 -7196.0 NULL NULL -30.0 7196.0 1557.8500000000004 1849372.0 NULL 1.329550715E9 7196.0 28.611 30.0 NULL