You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/06/21 07:25:26 UTC
[01/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Repository: hive
Updated Branches:
refs/heads/spark 58fafa2e4 -> 18990c834
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index c2035f2..657f1b5 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -148,14 +148,14 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((762 = cbigint) or ((csmallint < cfloat) and ((ctimestamp2 > -5) and (cdouble <> cint)))) or (cstring1 = 'a')) or ((cbigint <= -1.389) and ((cstring2 <> 'a') and ((79.553 <> cint) and (cboolean2 <> cboolean1))))) (type: boolean)
+ predicate: ((762 = cbigint) or (((UDFToFloat(csmallint) < cfloat) and ((UDFToDouble(ctimestamp2) > -5.0) and (cdouble <> UDFToDouble(cint)))) or ((cstring1 = 'a') or ((UDFToDouble(cbigint) <= -1.389) and ((cstring2 <> 'a') and ((79.553 <> UDFToDouble(cint)) and (cboolean2 <> cboolean1))))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdouble (type: double), csmallint (type: smallint), cfloat (type: float), ctinyint (type: tinyint)
- outputColumnNames: cint, cdouble, csmallint, cfloat, ctinyint
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cint), sum(cdouble), stddev_pop(cint), stddev_samp(csmallint), var_samp(cint), avg(cfloat), stddev_samp(cint), min(ctinyint), count(csmallint)
+ aggregations: avg(_col0), sum(_col1), stddev_pop(_col0), stddev_samp(_col2), var_samp(_col0), avg(_col3), stddev_samp(_col0), min(_col4), count(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
@@ -172,7 +172,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), (_col0 + -3728) (type: double), (- (_col0 + -3728)) (type: double), (- (- (_col0 + -3728))) (type: double), ((- (- (_col0 + -3728))) * (_col0 + -3728)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728))) * (_col0 + -3728)) * (- (- (_col0 + -3728)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728)))) (type: double), ((_col2 - (- (- (_col0 + -3728)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563) (type: double), _col6 (type: double), (- ((- _col2) / -563)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (_col7 / ((- _col2) / -563)) (type: double), (- (_col0 / _col1)) (type: double)
+ expressions: _col0 (type: double), (_col0 + -3728.0) (type: double), (- (_col0 + -3728.0)) (type: double), (- (- (_col0 + -3728.0))) (type: double), ((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) (type: double), _col1 (type: double), (- _col0) (type: double), _col2 (type: double), (((- (- (_col0 + -3728.0))) * (_col0 + -3728.0)) * (- (- (_col0 + -3728.0)))) (type: double), _col3 (type: double), (- _col2) (type: double), (_col2 - (- (- (_col0 + -3728.0)))) (type: double), ((_col2 - (- (- (_col0 + -3728.0)))) * _col2) (type: double), _col4 (type: double), _col5 (type: double), (10.175 - _col4) (type: double), (- (10.175 - _col4)) (type: double), ((- _col2) / -563.0) (type: double), _col6 (type: double), (- ((- _col2) / -563.0)) (type: double), (_col0 / _col1) (type: double), _col7 (type: tinyint), _col8 (type: bigint), (UDFToDouble(_col7) / ((- _col2) / -563.0)) (type: double), (- (_col0 / _col1)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -360,14 +360,14 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((cbigint <= 197) and (cint < cbigint)) or ((cdouble >= -26.28) and (csmallint > cdouble))) or ((ctinyint > cfloat) and cstring1 regexp '.*ss.*')) or ((cfloat > 79.553) and (cstring2 like '10%'))) (type: boolean)
+ predicate: (((cbigint <= 197) and (UDFToLong(cint) < cbigint)) or (((cdouble >= -26.28) and (UDFToDouble(csmallint) > cdouble)) or (((UDFToFloat(ctinyint) > cfloat) and cstring1 regexp '.*ss.*') or ((cfloat > 79.553) and (cstring2 like '10%'))))) (type: boolean)
Statistics: Num rows: 6826 Data size: 209555 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), csmallint (type: smallint), cdouble (type: double), ctinyint (type: tinyint)
- outputColumnNames: cint, cbigint, csmallint, cdouble, ctinyint
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 6826 Data size: 209555 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cint), var_pop(cbigint), stddev_pop(csmallint), max(cdouble), avg(ctinyint), min(cint), min(cdouble), stddev_samp(csmallint), var_samp(cint)
+ aggregations: max(_col0), var_pop(_col1), stddev_pop(_col2), max(_col3), avg(_col4), min(_col0), min(_col3), stddev_samp(_col2), var_samp(_col0)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -384,7 +384,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), (_col0 / -3728) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), ((_col0 * -3728) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (_col0 / -3728)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((_col0 / -3728) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double)
+ expressions: _col0 (type: int), (UDFToDouble(_col0) / -3728.0) (type: double), (_col0 * -3728) (type: int), _col1 (type: double), (- (_col0 * -3728)) (type: int), _col2 (type: double), (-563 % (_col0 * -3728)) (type: int), (_col1 / _col2) (type: double), (- _col2) (type: double), _col3 (type: double), _col4 (type: double), (_col2 - 10.175) (type: double), _col5 (type: int), (UDFToDouble((_col0 * -3728)) % (_col2 - 10.175)) (type: double), (- _col3) (type: double), _col6 (type: double), (_col3 % -26.28) (type: double), _col7 (type: double), (- (UDFToDouble(_col0) / -3728.0)) (type: double), ((- (_col0 * -3728)) % (-563 % (_col0 * -3728))) (type: int), ((UDFToDouble(_col0) / -3728.0) - _col4) (type: double), (- (_col0 * -3728)) (type: int), _col8 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -563,14 +563,14 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((ctimestamp1 = ctimestamp2) or (762.0 = cfloat)) or (cstring1 = 'ss')) or ((csmallint <= cbigint) and (1 = cboolean2))) or (cboolean1 is not null and (ctimestamp2 is not null and (cstring2 > 'a')))) (type: boolean)
+ predicate: ((ctimestamp1 = ctimestamp2) or ((762.0 = cfloat) or ((cstring1 = 'ss') or (((UDFToLong(csmallint) <= cbigint) and (1 = cboolean2)) or (cboolean1 is not null and (ctimestamp2 is not null and (cstring2 > 'a'))))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double)
- outputColumnNames: cbigint, ctinyint, csmallint, cint, cdouble
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: var_pop(cbigint), count(), max(ctinyint), stddev_pop(csmallint), max(cint), stddev_samp(cdouble), count(ctinyint), avg(ctinyint)
+ aggregations: var_pop(_col0), count(), max(_col1), stddev_pop(_col2), max(_col3), stddev_samp(_col4), count(_col1), avg(_col1)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -587,7 +587,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (_col1 % 79.553) (type: double), _col2 (type: tinyint), (_col1 - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (_col2 + (762 * (- _col1))) (type: bigint), ((- _col0) + _col4) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (_col2 + (762 * (- _col1)))) (type: bigint)
+ expressions: _col0 (type: double), (- _col0) (type: double), (_col0 - (- _col0)) (type: double), _col1 (type: bigint), (UDFToDouble(_col1) % 79.553) (type: double), _col2 (type: tinyint), (UDFToDouble(_col1) - (- _col0)) (type: double), (- (- _col0)) (type: double), (-1.0 % (- _col0)) (type: double), _col1 (type: bigint), (- _col1) (type: bigint), _col3 (type: double), (- (- (- _col0))) (type: double), (762 * (- _col1)) (type: bigint), _col4 (type: int), (UDFToLong(_col2) + (762 * (- _col1))) (type: bigint), ((- _col0) + UDFToDouble(_col4)) (type: double), _col5 (type: double), ((- _col1) % _col1) (type: bigint), _col6 (type: bigint), _col7 (type: double), (-3728 % (UDFToLong(_col2) + (762 * (- _col1)))) (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -745,14 +745,14 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((ctimestamp2 <= ctimestamp1) and ((cbigint <> cdouble) and ('ss' <= cstring1))) or ((csmallint < ctinyint) and (ctimestamp1 >= 0))) or (cfloat = 17.0)) (type: boolean)
+ predicate: (((ctimestamp2 <= ctimestamp1) and ((UDFToDouble(cbigint) <> cdouble) and ('ss' <= cstring1))) or (((csmallint < UDFToShort(ctinyint)) and (UDFToDouble(ctimestamp1) >= 0.0)) or (cfloat = 17.0))) (type: boolean)
Statistics: Num rows: 8874 Data size: 272428 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cbigint (type: bigint), cint (type: int), cfloat (type: float)
- outputColumnNames: ctinyint, cbigint, cint, cfloat
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 8874 Data size: 272428 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(ctinyint), max(cbigint), stddev_samp(cint), var_pop(cint), var_pop(cbigint), max(cfloat)
+ aggregations: avg(_col0), max(_col1), stddev_samp(_col2), var_pop(_col2), var_pop(_col1), max(_col3)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -769,7 +769,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), (_col0 + 6981) (type: double), ((_col0 + 6981) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981) + _col0) / _col0) (type: double), (- (_col0 + 6981)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), ((- _col1) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double)
+ expressions: _col0 (type: double), (_col0 + 6981.0) (type: double), ((_col0 + 6981.0) + _col0) (type: double), _col1 (type: bigint), (((_col0 + 6981.0) + _col0) / _col0) (type: double), (- (_col0 + 6981.0)) (type: double), _col2 (type: double), (_col0 % (- (_col0 + 6981.0))) (type: double), _col3 (type: double), _col4 (type: double), (- _col1) (type: bigint), (UDFToDouble((- _col1)) / _col2) (type: double), _col5 (type: float), (_col4 * -26.28) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -935,10 +935,10 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((cstring1 regexp 'a.*' and (cstring2 like '%ss%')) or ((1 <> cboolean2) and ((csmallint < 79.553) and (-257 <> ctinyint)))) or ((cdouble > ctinyint) and (cfloat >= cint))) or ((cint < cbigint) and (ctinyint > cbigint))) (type: boolean)
+ predicate: ((cstring1 regexp 'a.*' and (cstring2 like '%ss%')) or (((1 <> cboolean2) and ((UDFToDouble(csmallint) < 79.553) and (-257 <> UDFToInteger(ctinyint)))) or (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint))))) (type: boolean)
Statistics: Num rows: 9898 Data size: 303864 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728 * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - cint) (type: double), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (cint / cint) (type: double), ((-863.257 - cint) - -26.28) (type: double), (- cfloat) (type: float), (cdouble * -89010) (type: double), (ctinyint / 988888) (type: double), (- ctinyint) (type: tinyint), (79.553 / ctinyint) (type: double)
+ expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728 * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - UDFToDouble(cint)) (type: double), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - UDFToDouble(cint)) - -26.28) (type: double), (- cfloat) (type: float), (cdouble * -89010.0) (type: double), (UDFToDouble(ctinyint) / 988888.0) (type: double), (- ctinyint) (type: tinyint), (79.553 / UDFToDouble(ctinyint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22
Statistics: Num rows: 9898 Data size: 303864 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -1194,10 +1194,10 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((197.0 > ctinyint) and (cint = cbigint)) or (cbigint = 359)) or (cboolean1 < 0)) or ((cstring1 like '%ss') and (cfloat <= ctinyint))) (type: boolean)
+ predicate: (((197.0 > UDFToDouble(ctinyint)) and (UDFToLong(cint) = cbigint)) or ((cbigint = 359) or ((cboolean1 < 0) or ((cstring1 like '%ss') and (cfloat <= UDFToFloat(ctinyint)))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), cbigint (type: bigint), (cint / cbigint) (type: double), (cbigint % 79.553) (type: double), (- (cint / cbigint)) (type: double), (10.175 % cfloat) (type: double), (- cfloat) (type: float), (cfloat - (- cfloat)) (type: float), ((cfloat - (- cfloat)) % -6432) (type: float), (cdouble * csmallint) (type: double), (- cdouble) (type: double), (- cbigint) (type: bigint), cstring1 (type: string), (cfloat - (cint / cbigint)) (type: double), (- csmallint) (type: smallint), (3569 % cbigint) (type: bigint), (359 - cdouble) (type: double), cboolean1 (type: boolean), cfloat (type: float), cdouble (type: double), ctimestamp2 (type: timestamp), csmallint (type: smallint), cstring2 (type: string), cboolean2 (type: boolean)
+ expressions: cint (type: int), cbigint (type: bigint), (UDFToDouble(cint) / UDFToDouble(cbigint)) (type: double), (UDFToDouble(cbigint) % 79.553) (type: double), (- (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (10.175 % UDFToDouble(cfloat)) (type: double), (- cfloat) (type: float), (cfloat - (- cfloat)) (type: float), ((cfloat - (- cfloat)) % -6432.0) (type: float), (cdouble * UDFToDouble(csmallint)) (type: double), (- cdouble) (type: double), (- cbigint) (type: bigint), cstring1 (type: string), (UDFToDouble(cfloat) - (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (- csmallint) (type: smallint), (3569 % cbigint) (type: bigint), (359.0 - cdouble) (type: double), cboolean1 (type: boolean), cfloat (type: float), cdouble (type: double), ctimestamp2 (type: timestamp), csmallint (type: smallint), cstring2 (type: string), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col21, _col22, _col23, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -1402,10 +1402,10 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((csmallint > -26.28) and (cstring2 like 'ss')) or ((cdouble <= cbigint) and ((cstring1 >= 'ss') and (cint <> cdouble)))) or (ctinyint = -89010)) or ((cbigint <= cfloat) and (-26.28 <= csmallint))) (type: boolean)
+ predicate: (((UDFToDouble(csmallint) > -26.28) and (cstring2 like 'ss')) or (((cdouble <= UDFToDouble(cbigint)) and ((cstring1 >= 'ss') and (UDFToDouble(cint) <> cdouble))) or ((UDFToInteger(ctinyint) = -89010) or ((UDFToFloat(cbigint) <= cfloat) and (-26.28 <= UDFToDouble(csmallint)))))) (type: boolean)
Statistics: Num rows: 10922 Data size: 335301 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cint (type: int), cstring1 (type: string), cboolean2 (type: boolean), ctimestamp2 (type: timestamp), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), cboolean1 (type: boolean), (cint + csmallint) (type: int), (cbigint - ctinyint) (type: bigint), (- cbigint) (type: bigint), (- cfloat) (type: float), ((cbigint - ctinyint) + cbigint) (type: bigint), (cdouble / cdouble) (type: double), (- cdouble) (type: double), ((cint + csmallint) * (- cbigint)) (type: bigint), ((- cdouble) + cbigint) (type: double), (-1.389 / ctinyint) (type: double), (cbigint % cdouble) (type: double), (- csmallint) (type: smallint), (csmallint + (cint + csmallint)) (type: int)
+ expressions: cint (type: int), cstring1 (type: string), cboolean2 (type: boolean), ctimestamp2 (type: timestamp), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), cboolean1 (type: boolean), (cint + UDFToInteger(csmallint)) (type: int), (cbigint - UDFToLong(ctinyint)) (type: bigint), (- cbigint) (type: bigint), (- cfloat) (type: float), ((cbigint - UDFToLong(ctinyint)) + cbigint) (type: bigint), (cdouble / cdouble) (type: double), (- cdouble) (type: double), (UDFToLong((cint + UDFToInteger(csmallint))) * (- cbigint)) (type: bigint), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (-1.389 / UDFToDouble(ctinyint)) (type: double), (UDFToDouble(cbigint) % cdouble) (type: double), (- csmallint) (type: smallint), (UDFToInteger(csmallint) + (cint + UDFToInteger(csmallint))) (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
Statistics: Num rows: 10922 Data size: 335301 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -1668,10 +1668,10 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((-1.389 >= cint) and ((csmallint < ctinyint) and (-6432 > csmallint))) or ((cdouble >= cfloat) and (cstring2 <= 'a'))) or ((cstring1 like 'ss%') and (10.175 > cbigint))) (type: boolean)
+ predicate: (((-1.389 >= UDFToDouble(cint)) and ((csmallint < UDFToShort(ctinyint)) and (-6432 > UDFToInteger(csmallint)))) or (((cdouble >= UDFToDouble(cfloat)) and (cstring2 <= 'a')) or ((cstring1 like 'ss%') and (10.175 > UDFToDouble(cbigint))))) (type: boolean)
Statistics: Num rows: 3868 Data size: 118746 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ctimestamp1 (type: timestamp), cstring2 (type: string), (cdouble * 10.175) (type: double), ((-6432 * cfloat) / cfloat) (type: double), (- cfloat) (type: float), (cint % csmallint) (type: int), (cdouble * (- cdouble)) (type: double), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), (cbigint / 3569) (type: double), (-257 - csmallint) (type: int), (-6432 * cfloat) (type: float), (- cdouble) (type: double)
+ expressions: ctimestamp1 (type: timestamp), cstring2 (type: string), (cdouble * 10.175) (type: double), (UDFToDouble((-6432.0 * cfloat)) / UDFToDouble(cfloat)) (type: double), (- cfloat) (type: float), (cint % UDFToInteger(csmallint)) (type: int), (cdouble * (- cdouble)) (type: double), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), (UDFToDouble(cbigint) / 3569.0) (type: double), (-257 - UDFToInteger(csmallint)) (type: int), (-6432.0 * cfloat) (type: float), (- cdouble) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col15, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 3868 Data size: 118746 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -1876,15 +1876,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((csmallint >= -257) and ((-6432 = csmallint) or ((cint >= cdouble) and (ctinyint <= cint)))) (type: boolean)
+ predicate: ((UDFToInteger(csmallint) >= -257) and ((-6432 = UDFToInteger(csmallint)) or ((UDFToDouble(cint) >= cdouble) and (UDFToInteger(ctinyint) <= cint)))) (type: boolean)
Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), cbigint (type: bigint), ctinyint (type: tinyint)
- outputColumnNames: csmallint, cbigint, ctinyint
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: stddev_samp(csmallint), sum(cbigint), var_pop(ctinyint), count()
- keys: csmallint (type: smallint)
+ aggregations: stddev_samp(_col0), sum(_col1), var_pop(_col2), count()
+ keys: _col0 (type: smallint)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 2503 Data size: 76841 Basic stats: COMPLETE Column stats: NONE
@@ -1904,7 +1904,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1251 Data size: 38405 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: smallint), (_col0 % -75) (type: int), _col1 (type: double), (-1.389 / _col0) (type: double), _col2 (type: bigint), ((_col0 % -75) / _col2) (type: double), (- (_col0 % -75)) (type: int), _col3 (type: double), (- (- (_col0 % -75))) (type: int), _col4 (type: bigint), (_col4 - -89010) (type: bigint)
+ expressions: _col0 (type: smallint), (UDFToInteger(_col0) % -75) (type: int), _col1 (type: double), (-1.389 / UDFToDouble(_col0)) (type: double), _col2 (type: bigint), (UDFToDouble((UDFToInteger(_col0) % -75)) / UDFToDouble(_col2)) (type: double), (- (UDFToInteger(_col0) % -75)) (type: int), _col3 (type: double), (- (- (UDFToInteger(_col0) % -75))) (type: int), _col4 (type: bigint), (_col4 - -89010) (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1251 Data size: 38405 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -2084,15 +2084,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cdouble > 2563.58) and (((cbigint >= cint) and ((csmallint < cint) and (cfloat < -5638.15))) or ((cdouble <= cbigint) and (-5638.15 > cbigint)))) (type: boolean)
+ predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and ((UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15))) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), cfloat (type: float)
- outputColumnNames: cdouble, cfloat
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: var_samp(cdouble), count(cfloat), sum(cfloat), var_pop(cdouble), stddev_pop(cdouble), sum(cdouble)
- keys: cdouble (type: double)
+ aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
+ keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
@@ -2112,7 +2112,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
+ expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -2339,15 +2339,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ctimestamp1 <> 0) and (((((((-257 <> ctinyint) and cboolean2 is not null) and (cstring1 regexp '.*ss' and (-3 < ctimestamp1))) or (ctimestamp2 = -5)) or ((ctimestamp1 < 0) and (cstring2 like '%b%'))) or (cdouble = cint)) or (cboolean1 is null and (cfloat < cint)))) (type: boolean)
+ predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and (cboolean2 is not null and (cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))))) or ((UDFToDouble(ctimestamp2) = -5.0) or (((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or ((cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint)))))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cstring1 (type: string), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double)
- outputColumnNames: ctimestamp1, cstring1, cint, csmallint, ctinyint, cfloat, cdouble
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: stddev_pop(cint), avg(csmallint), count(), min(ctinyint), var_samp(csmallint), var_pop(cfloat), avg(cint), var_samp(cfloat), avg(cfloat), min(cdouble), var_pop(csmallint), stddev_pop(ctinyint), sum(cint)
- keys: ctimestamp1 (type: timestamp), cstring1 (type: string)
+ aggregations: stddev_pop(_col2), avg(_col3), count(), min(_col4), var_samp(_col3), var_pop(_col5), avg(_col2), var_samp(_col5), avg(_col5), min(_col6), var_pop(_col3), stddev_pop(_col4), sum(_col2)
+ keys: _col0 (type: timestamp), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
@@ -2367,7 +2367,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: timestamp), _col1 (type: string), ((-26.28 - _col2) * (- _col2)) (type: double), _col5 (type: tinyint), (((-26.28 - _col2) * (- _col2)) * (- _col4)) (type: double), (- (_col2 * 10.175)) (type: double), _col6 (type: double), (_col6 + (((-26.28 - _col2) * (- _col2)) * (- _col4))) (type: double), (- (- _col2)) (type: double), ((- _col4) / _col2) (type: double), _col7 (type: double), (10.175 / _col3) (type: double), _col2 (type: double), _col8 (type: double), _col9 (type: double), ((_col6 + (((-26.28 - _col2) * (- _col2)) * (- _col4))) - (((-26.28 - _col2) * (- _col2)) * (- _col4))) (type: double), (- (- (_col2 * 10.175))) (type: double), _col10 (type: double), (((_col6 + (((-26.28 - _col2) * (- _col2)) * (- _col4))) - (((-26.28 - _col2) * (- _col2)) * (- _col4))) * 10.175) (type: double), (10.175 % (10.175 / _col3)) (type: double), (- _col5) (type: tinyint), _col11 (type: double), _col12 (type: double), (_col2 * 10.175) (type: double), (- ((-
26.28 - _col2) * (- _col2))) (type: double), ((- _col2) % _col10) (type: double), (-26.28 / (- _col5)) (type: double), _col13 (type: double), _col14 (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * (- _col4))) / _col7) (type: double), (- (- _col4)) (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * (- _col4))) % -26.28) (type: double), (- _col2) (type: double), _col3 (type: double), (-26.28 - _col2) (type: double), _col4 (type: bigint), (- _col4) (type: bigint)
+ expressions: _col0 (type: timestamp), _col1 (type: string), ((-26.28 - _col2) * (- _col2)) (type: double), _col5 (type: tinyint), (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4))) (type: double), (- (_col2 * 10.175)) (type: double), _col6 (type: double), (_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) (type: double), (- (- _col2)) (type: double), (UDFToDouble((- _col4)) / _col2) (type: double), _col7 (type: double), (10.175 / _col3) (type: double), _col2 (type: double), _col8 (type: double), _col9 (type: double), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) - (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) (type: double), (- (- (_col2 * 10.175))) (type: double), _col10 (type: double), (((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) - (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) * 10.175) (type: double), (10.175 % (10.175 / _col3)) (type: double), (- _col5) (type: tiny
int), _col11 (type: double), _col12 (type: double), (_col2 * 10.175) (type: double), (- ((-26.28 - _col2) * (- _col2))) (type: double), ((- _col2) % _col10) (type: double), (-26.28 / UDFToDouble((- _col5))) (type: double), _col13 (type: double), _col14 (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) / _col7) (type: double), (- (- _col4)) (type: bigint), ((_col6 + (((-26.28 - _col2) * (- _col2)) * UDFToDouble((- _col4)))) % -26.28) (type: double), (- _col2) (type: double), _col3 (type: double), (-26.28 - _col2) (type: double), _col4 (type: bigint), (- _col4) (type: bigint)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col3, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col38, _col4, _col5, _col7, _col8, _col9
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -2672,15 +2672,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cboolean1 is not null and (((((cdouble < csmallint) and ((cboolean2 = cboolean1) and (cbigint <= -863.257))) or ((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1)))) or cstring2 regexp 'b') or ((csmallint >= ctinyint) and ctimestamp2 is null))) (type: boolean)
+ predicate: (cboolean1 is not null and (((cdouble < UDFToDouble(csmallint)) and ((cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257))) or (((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1))) or (cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null))))) (type: boolean)
Statistics: Num rows: 4778 Data size: 146682 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint)
- outputColumnNames: cboolean1, cfloat, cbigint, cint, cdouble, ctinyint, csmallint
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 4778 Data size: 146682 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(cfloat), sum(cbigint), var_samp(cint), avg(cdouble), min(cbigint), var_pop(cbigint), sum(cint), stddev_samp(ctinyint), stddev_pop(csmallint), avg(cint)
- keys: cboolean1 (type: boolean)
+ aggregations: max(_col1), sum(_col2), var_samp(_col3), avg(_col4), min(_col2), var_pop(_col2), sum(_col3), stddev_samp(_col5), stddev_pop(_col6), avg(_col3)
+ keys: _col0 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 4778 Data size: 146682 Basic stats: COMPLETE Column stats: NONE
@@ -2700,7 +2700,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 2389 Data size: 73341 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: float), ((_col2 - 10.175) + _col3) (type: double), _col5 (type: bigint), _col6 (type: double), (- (10.175 + (- _col1))) (type: double), (79.553 / _col6) (type: double), (_col3 % (79.553 / _col6)) (type: double), _col7 (type: bigint), _col8 (type: double), (-1.389 * _col5) (type: double), (- _col1) (type: float), (_col7 - (-1.389 * _col5)) (type: double), _col9 (type: double), (- (_col7 - (-1.389 * _col5))) (type: double), _col10 (type: double), (- _col10) (type: double), (_col10 * _col7) (type: double), (-26.28 / _col1) (type: double), _col2 (type: bigint), (_col2 - 10.175) (type: double), _col3 (type: double), (_col3 % _col1) (type: double), (10.175 + (- _col1)) (type: double), _col4 (type: double)
+ expressions: _col0 (type: boolean), _col1 (type: float), ((UDFToDouble(_col2) - 10.175) + _col3) (type: double), _col5 (type: bigint), _col6 (type: double), (- (10.175 + UDFToDouble((- _col1)))) (type: double), (79.553 / _col6) (type: double), (_col3 % (79.553 / _col6)) (type: double), _col7 (type: bigint), _col8 (type: double), (-1.389 * UDFToDouble(_col5)) (type: double), (- _col1) (type: float), (UDFToDouble(_col7) - (-1.389 * UDFToDouble(_col5))) (type: double), _col9 (type: double), (- (UDFToDouble(_col7) - (-1.389 * UDFToDouble(_col5)))) (type: double), _col10 (type: double), (- _col10) (type: double), (_col10 * UDFToDouble(_col7)) (type: double), (-26.28 / UDFToDouble(_col1)) (type: double), _col2 (type: bigint), (UDFToDouble(_col2) - 10.175) (type: double), _col3 (type: double), (_col3 % UDFToDouble(_col1)) (type: double), (10.175 + UDFToDouble((- _col1))) (type: double), _col4 (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col15, _col17, _col18, _col19, _col2, _col20, _col21, _col22, _col23, _col24, _col25, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 2389 Data size: 73341 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -2915,10 +2915,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: i (type: int)
- outputColumnNames: i
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: count(i)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -3093,10 +3093,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
- outputColumnNames: ctinyint
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(ctinyint)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -3160,10 +3160,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
- outputColumnNames: cint
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cint)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -3227,10 +3227,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float)
- outputColumnNames: cfloat
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cfloat)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -3294,10 +3294,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
- outputColumnNames: cstring1
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cstring1)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -3361,10 +3361,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean)
- outputColumnNames: cboolean1
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cboolean1)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
index daf6ad3..0e1e9d3 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out
@@ -46,7 +46,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((csmallint = 418) or (csmallint = 12205)) or (csmallint = 10583)) (type: boolean)
+ predicate: ((csmallint = 418) or ((csmallint = 12205) or (csmallint = 10583))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END (type: string), CASE (csmallint) WHEN (418) THEN ('a') WHEN (12205) THEN ('b') ELSE ('c') END (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out b/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
index 78b0b37..b6c2b35 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_mapjoin.q.out
@@ -23,15 +23,19 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: t2
+ alias: t1
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: cint is not null (type: boolean)
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 cint (type: int)
- 1 cint (type: int)
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -49,25 +53,33 @@ STAGE PLANS:
Filter Operator
predicate: cint is not null (type: boolean)
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 cint (type: int)
- 1 cint (type: int)
- outputColumnNames: _col2, _col17
- input vertices:
- 1 Map 3
- Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(_col2), max(_col17), min(_col2), avg((_col2 + _col17))
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), (_col0 + _col1) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Local Work:
Map Reduce Local Work
Execution mode: vectorized
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out b/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
index b84a4c3..881641f 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_shufflejoin.q.out
@@ -31,25 +31,33 @@ STAGE PLANS:
Filter Operator
predicate: cint is not null (type: boolean)
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: cint (type: int)
- sort order: +
- Map-reduce partition columns: cint (type: int)
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 5
Map Operator Tree:
TableScan
- alias: t2
+ alias: t1
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: cint is not null (type: boolean)
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: cint (type: int)
- sort order: +
- Map-reduce partition columns: cint (type: int)
+ Select Operator
+ expressions: cint (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -57,19 +65,23 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 cint (type: int)
- 1 cint (type: int)
- outputColumnNames: _col2, _col17
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(_col2), max(_col17), min(_col2), avg((_col2 + _col17))
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), (_col0 + _col1) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 6758 Data size: 207479 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0), max(_col1), min(_col0), avg(_col2)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: int), _col2 (type: int), _col3 (type: struct<count:bigint,sum:double,input:int>)
Reducer 3
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
index ed1ba4b..290db0d 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
@@ -644,10 +644,10 @@ STAGE PLANS:
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp)
- outputColumnNames: ctimestamp1
+ outputColumnNames: _col0
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: min(ctimestamp1), max(ctimestamp1), count(ctimestamp1), count()
+ aggregations: min(_col0), max(_col0), count(_col0), count()
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: NONE
@@ -725,10 +725,10 @@ STAGE PLANS:
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp)
- outputColumnNames: ctimestamp1
+ outputColumnNames: _col0
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(ctimestamp1)
+ aggregations: sum(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -815,10 +815,10 @@ STAGE PLANS:
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp)
- outputColumnNames: ctimestamp1
+ outputColumnNames: _col0
Statistics: Num rows: 40 Data size: 84 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(ctimestamp1), variance(ctimestamp1), var_pop(ctimestamp1), var_samp(ctimestamp1), std(ctimestamp1), stddev(ctimestamp1), stddev_pop(ctimestamp1), stddev_samp(ctimestamp1)
+ aggregations: avg(_col0), variance(_col0), var_pop(_col0), var_samp(_col0), std(_col0), stddev(_col0), stddev_pop(_col0), stddev_samp(_col0)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/tez/cbo_subq_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/cbo_subq_in.q.out b/ql/src/test/results/clientpositive/tez/cbo_subq_in.q.out
index c1f3de7..f6bfad2 100644
--- a/ql/src/test/results/clientpositive/tez/cbo_subq_in.q.out
+++ b/ql/src/test/results/clientpositive/tez/cbo_subq_in.q.out
@@ -69,6 +69,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
PREHOOK: type: QUERY
PREHOOK: Input: default@lineitem
#### A masked pattern was here ####
@@ -77,6 +78,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
POSTHOOK: type: QUERY
POSTHOOK: Input: default@lineitem
#### A masked pattern was here ####
[04/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/skewjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/skewjoin.q.out b/ql/src/test/results/clientpositive/spark/skewjoin.q.out
index 5fb2ecb..ec74786 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoin.q.out
@@ -105,25 +105,33 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -131,12 +139,12 @@ STAGE PLANS:
Inner Join 0 to 1
handleSkewJoin: true
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -178,9 +186,9 @@ STAGE PLANS:
keys:
0 reducesinkkey0 (type: string)
1 reducesinkkey0 (type: string)
- outputColumnNames: _col0, _col6
+ outputColumnNames: _col1, _col2
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -218,7 +226,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(key)), sum(hash(value)) FROM dest_j1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
@@ -625,14 +633,15 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 4
Map Operator Tree:
TableScan
@@ -642,15 +651,14 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -660,19 +668,23 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col2, _col3
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col2)), sum(hash(_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col0) (type: int), hash(_col1) (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-4
Conditional Operator
@@ -704,18 +716,22 @@ STAGE PLANS:
keys:
0 reducesinkkey0 (type: string)
1 reducesinkkey0 (type: string)
- outputColumnNames: _col2, _col3
- Group By Operator
- aggregations: sum(hash(_col2)), sum(hash(_col3))
- mode: hash
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: hash(_col0) (type: int), hash(_col1) (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Local Work:
Map Reduce Local Work
@@ -824,16 +840,16 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key is not null and (substring(value, 5) + 1) is not null) (type: boolean)
+ predicate: (key is not null and (UDFToDouble(substring(value, 5)) + 1.0) is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), (substring(_col1, 5) + 1) (type: double)
+ key expressions: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double)
sort order: ++
- Map-reduce partition columns: _col0 (type: string), (substring(_col1, 5) + 1) (type: double)
+ Map-reduce partition columns: _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
@@ -844,20 +860,24 @@ STAGE PLANS:
handleSkewJoin: true
keys:
0 _col0 (type: string), UDFToDouble(substring(_col1, 5)) (type: double)
- 1 _col0 (type: string), (substring(_col1, 5) + 1) (type: double)
+ 1 _col0 (type: string), (UDFToDouble(substring(_col1, 5)) + 1.0) (type: double)
outputColumnNames: _col2, _col3
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col2)), sum(hash(_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col2) (type: int), hash(_col3) (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-4
Conditional Operator
@@ -890,17 +910,21 @@ STAGE PLANS:
0 reducesinkkey0 (type: string), reducesinkkey1 (type: double)
1 reducesinkkey0 (type: string), reducesinkkey1 (type: double)
outputColumnNames: _col2, _col3
- Group By Operator
- aggregations: sum(hash(_col2)), sum(hash(_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col2) (type: int), hash(_col3) (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), sum(_col1)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out b/ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out
index 8afc656..4025885 100644
--- a/ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out
+++ b/ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out
@@ -29,25 +29,33 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 4
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -55,8 +63,8 @@ STAGE PLANS:
Inner Join 0 to 1
handleSkewJoin: true
keys:
- 0 key (type: string)
- 1 key (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/stats1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/stats1.q.out b/ql/src/test/results/clientpositive/spark/stats1.q.out
index f00db10..ec2edc4 100644
--- a/ql/src/test/results/clientpositive/spark/stats1.q.out
+++ b/ql/src/test/results/clientpositive/spark/stats1.q.out
@@ -121,7 +121,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@tmptable
POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable.value EXPRESSION [(src1)s2.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT * FROM tmptable x SORT BY x.key, x.value
PREHOOK: type: QUERY
PREHOOK: Input: default@tmptable
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/stats_only_null.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/stats_only_null.q.out b/ql/src/test/results/clientpositive/spark/stats_only_null.q.out
index 2840729..cceceef 100644
--- a/ql/src/test/results/clientpositive/spark/stats_only_null.q.out
+++ b/ql/src/test/results/clientpositive/spark/stats_only_null.q.out
@@ -90,10 +90,10 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: a (type: double), b (type: int), c (type: string), d (type: smallint)
- outputColumnNames: a, b, c, d
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(), count(a), count(b), count(c), count(d)
+ aggregations: count(), count(_col0), count(_col1), count(_col2), count(_col3)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE
@@ -146,10 +146,10 @@ STAGE PLANS:
Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: a (type: double), b (type: int), c (type: string), d (type: smallint)
- outputColumnNames: a, b, c, d
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(), count(a), count(b), count(c), count(d)
+ aggregations: count(), count(_col0), count(_col1), count(_col2), count(_col3)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
index 854ca14..28eda26 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
@@ -43,25 +43,29 @@ STAGE PLANS:
Filter Operator
predicate: (value is not null and key is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string), key (type: string)
- sort order: ++
- Map-reduce partition columns: value (type: string), key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((value > 'val_9') and key is not null) (type: boolean)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), key (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: _col1 (type: string), _col2 (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
@@ -76,7 +80,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 value (type: string), key (type: string)
+ 0 _col1 (type: string), _col0 (type: string)
1 _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 6cf1ae4..00b3399 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -33,16 +33,20 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: s1
+ alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > '9') (type: boolean)
@@ -67,7 +71,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
@@ -147,15 +151,19 @@ STAGE PLANS:
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key > '9') and value is not null) (type: boolean)
@@ -180,7 +188,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 key (type: string), value (type: string)
+ 0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
@@ -267,15 +275,19 @@ STAGE PLANS:
TableScan
alias: part
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: UDFToDouble(p_size) is not null (type: boolean)
- Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: UDFToDouble(p_size) (type: double)
- sort order: +
- Map-reduce partition columns: UDFToDouble(p_size) (type: double)
+ Select Operator
+ expressions: p_name (type: string), p_size (type: int), UDFToDouble(p_size) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col2 is not null (type: boolean)
Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_name (type: string), p_size (type: int)
+ Reduce Output Operator
+ key expressions: _col2 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: double)
+ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -292,21 +304,17 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 UDFToDouble(p_size) (type: double)
+ 0 _col2 (type: double)
1 _col0 (type: double)
- outputColumnNames: _col1, _col5
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col5 (type: int)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Reducer 4
Reduce Operator Tree:
Select Operator
@@ -438,16 +446,20 @@ STAGE PLANS:
Filter Operator
predicate: (p_size is not null and p_mfgr is not null) (type: boolean)
Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: p_size (type: int), p_mfgr (type: string)
- sort order: ++
- Map-reduce partition columns: p_size (type: int), p_mfgr (type: string)
+ Select Operator
+ expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_name (type: string)
+ Reduce Output Operator
+ key expressions: _col2 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col2 (type: int), _col1 (type: string)
+ Statistics: Num rows: 7 Data size: 847 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: part
+ alias: b
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: p_mfgr (type: string), p_size (type: int)
@@ -460,12 +472,12 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 p_size (type: int), p_mfgr (type: string)
+ 0 _col2 (type: int), _col1 (type: string)
1 _col0 (type: int), _col1 (type: string)
- outputColumnNames: _col1, _col2, _col5
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int)
+ expressions: _col1 (type: string), _col0 (type: string), _col2 (type: int)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 7 Data size: 931 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -617,36 +629,44 @@ STAGE PLANS:
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key > '9') and value is not null) (type: boolean)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: key (type: string), value (type: string)
- mode: hash
+ Select Operator
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 key (type: string), value (type: string)
+ 0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
@@ -756,9 +776,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
- Reducer 4 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 6 <- Map 5 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -767,33 +787,19 @@ STAGE PLANS:
alias: lineitem
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: l_partkey is not null (type: boolean)
- Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: l_partkey (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Map 5
- Map Operator Tree:
- TableScan
- alias: li
- Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((l_partkey is not null and l_orderkey is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: l_partkey (type: int)
- sort order: +
- Map-reduce partition columns: l_partkey (type: int)
- Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
- value expressions: l_orderkey (type: int), l_suppkey (type: int)
- Map 6
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int)
+ Map 4
Map Operator Tree:
TableScan
alias: lineitem
@@ -815,46 +821,56 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: lineitem
+ Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: l_partkey is not null (type: boolean)
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_partkey (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: int)
- 1 l_partkey (type: int)
- outputColumnNames: _col0, _col1, _col3
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col3 (type: int)
- Reducer 4
+ value expressions: _col2 (type: int)
+ Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
- Left Semi Join 0 to 1
+ Inner Join 0 to 1
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col3
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col3 (type: int)
+ expressions: _col1 (type: int), _col2 (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -864,6 +880,18 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 6
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out b/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out
index 92a8595..1bfdba2 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out
@@ -237,19 +237,19 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col0 = 0) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 0 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-1
Move Operator
@@ -580,16 +580,16 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col0 = 0) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 0 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
index 3660f1a..7576b48 100644
--- a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
+++ b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out
@@ -22,7 +22,7 @@ SELECT key, count(1) FROM T1 GROUP BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key
@@ -35,7 +35,7 @@ PREHOOK: query: SELECT key, val, count(1) FROM T1 GROUP BY key, val
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key,val
@@ -77,7 +77,7 @@ SELECT 1, key, count(1) FROM T1 GROUP BY 1, key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key
@@ -90,7 +90,7 @@ PREHOOK: query: SELECT key, 1, val, count(1) FROM T1 GROUP BY key, 1, val
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key,val
@@ -104,7 +104,7 @@ PREHOOK: query: SELECT key, 1, val, 2, count(1) FROM T1 GROUP BY key, 1, val, 2
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key,val
@@ -130,7 +130,7 @@ group by key + key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key
@@ -148,11 +148,11 @@ SELECT key, count(1) as c FROM T1 GROUP BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_2
+Operator:GBY_3
Table:default@t1
Keys:key
-Operator:GBY_8
+Operator:GBY_10
Table:default@t1
Keys:key
@@ -242,7 +242,7 @@ GROUP BY key, constant3, val
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
#### A masked pattern was here ####
-Operator:GBY_4
+Operator:GBY_3
Table:default@t1
Keys:key,val
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/temp_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/temp_table.q.out b/ql/src/test/results/clientpositive/spark/temp_table.q.out
index 0801abc..65e256d 100644
--- a/ql/src/test/results/clientpositive/spark/temp_table.q.out
+++ b/ql/src/test/results/clientpositive/spark/temp_table.q.out
@@ -19,7 +19,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 2) = 0) (type: boolean)
+ predicate: ((UDFToDouble(key) % 2.0) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -85,7 +85,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 2) = 1) (type: boolean)
+ predicate: ((UDFToDouble(key) % 2.0) = 1.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union.q.out b/ql/src/test/results/clientpositive/spark/union.q.out
index 2644473..a78504f 100644
--- a/ql/src/test/results/clientpositive/spark/union.q.out
+++ b/ql/src/test/results/clientpositive/spark/union.q.out
@@ -35,7 +35,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -54,7 +54,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key > 100) (type: boolean)
+ predicate: (UDFToDouble(key) > 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union10.q.out b/ql/src/test/results/clientpositive/spark/union10.q.out
index 9698dda..5e8fe38 100644
--- a/ql/src/test/results/clientpositive/spark/union10.q.out
+++ b/ql/src/test/results/clientpositive/spark/union10.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -79,7 +79,7 @@ STAGE PLANS:
Map 5
Map Operator Tree:
TableScan
- alias: s3
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -194,7 +194,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@tmptable
POSTHOOK: Lineage: tmptable.key EXPRESSION []
-POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src)s2.null, (src)s3.null, ]
+POSTHOOK: Lineage: tmptable.value EXPRESSION []
PREHOOK: query: select * from tmptable x sort by x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tmptable
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union11.q.out b/ql/src/test/results/clientpositive/spark/union11.q.out
index 300bad4..20c27c7 100644
--- a/ql/src/test/results/clientpositive/spark/union11.q.out
+++ b/ql/src/test/results/clientpositive/spark/union11.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -67,7 +67,7 @@ STAGE PLANS:
Map 6
Map Operator Tree:
TableScan
- alias: s3
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -91,22 +91,18 @@ STAGE PLANS:
expressions: 'tst1' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
@@ -133,22 +129,18 @@ STAGE PLANS:
expressions: 'tst2' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 7
Reduce Operator Tree:
Group By Operator
@@ -160,22 +152,18 @@ STAGE PLANS:
expressions: 'tst3' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union12.q.out b/ql/src/test/results/clientpositive/spark/union12.q.out
index b9ddfc8..cf2c7b7 100644
--- a/ql/src/test/results/clientpositive/spark/union12.q.out
+++ b/ql/src/test/results/clientpositive/spark/union12.q.out
@@ -198,7 +198,7 @@ POSTHOOK: Input: default@src1
POSTHOOK: Input: default@srcbucket
POSTHOOK: Output: default@tmptable
POSTHOOK: Lineage: tmptable.key EXPRESSION []
-POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.null, (srcbucket)s3.null, ]
+POSTHOOK: Lineage: tmptable.value EXPRESSION []
PREHOOK: query: select * from tmptable x sort by x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tmptable
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union13.q.out b/ql/src/test/results/clientpositive/spark/union13.q.out
index fca251c..0297988 100644
--- a/ql/src/test/results/clientpositive/spark/union13.q.out
+++ b/ql/src/test/results/clientpositive/spark/union13.q.out
@@ -40,7 +40,7 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union14.q.out b/ql/src/test/results/clientpositive/spark/union14.q.out
index 77051df..0c9542b 100644
--- a/ql/src/test/results/clientpositive/spark/union14.q.out
+++ b/ql/src/test/results/clientpositive/spark/union14.q.out
@@ -37,22 +37,18 @@ STAGE PLANS:
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Map 3
Map Operator Tree:
TableScan
@@ -95,22 +91,18 @@ STAGE PLANS:
expressions: 'tst1' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union15.q.out b/ql/src/test/results/clientpositive/spark/union15.q.out
index c51dd74..6be13c9 100644
--- a/ql/src/test/results/clientpositive/spark/union15.q.out
+++ b/ql/src/test/results/clientpositive/spark/union15.q.out
@@ -55,47 +55,39 @@ STAGE PLANS:
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 51 Data size: 470 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Map 5
Map Operator Tree:
TableScan
- alias: s3
+ alias: s2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 51 Data size: 470 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -107,22 +99,18 @@ STAGE PLANS:
expressions: 'tst1' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 51 Data size: 470 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union2.q.out b/ql/src/test/results/clientpositive/spark/union2.q.out
index 881ce21..e4afb1b 100644
--- a/ql/src/test/results/clientpositive/spark/union2.q.out
+++ b/ql/src/test/results/clientpositive/spark/union2.q.out
@@ -44,7 +44,7 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union20.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union20.q.out b/ql/src/test/results/clientpositive/spark/union20.q.out
index 546355f..6f0dca6 100644
--- a/ql/src/test/results/clientpositive/spark/union20.q.out
+++ b/ql/src/test/results/clientpositive/spark/union20.q.out
@@ -58,10 +58,10 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -76,7 +76,7 @@ STAGE PLANS:
Map 5
Map Operator Tree:
TableScan
- alias: s3
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -92,10 +92,10 @@ STAGE PLANS:
Map 7
Map Operator Tree:
TableScan
- alias: s4
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union24.q.out b/ql/src/test/results/clientpositive/spark/union24.q.out
index 7a05198..658ff9e 100644
--- a/ql/src/test/results/clientpositive/spark/union24.q.out
+++ b/ql/src/test/results/clientpositive/spark/union24.q.out
@@ -198,7 +198,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), count (type: bigint)
@@ -282,7 +282,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), count (type: bigint)
@@ -366,7 +366,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), count (type: bigint)
@@ -450,22 +450,26 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 10) (type: boolean)
+ predicate: (UDFToDouble(key) < 10.0) (type: boolean)
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: bigint)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 103 Data size: 494 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union25.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out
index db7dfc5..5193c06 100644
--- a/ql/src/test/results/clientpositive/spark/union25.q.out
+++ b/ql/src/test/results/clientpositive/spark/union25.q.out
@@ -100,43 +100,35 @@ STAGE PLANS:
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Map 5
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Group By Operator
[13/61] [abbrv] hive git commit: HIVE-10971 : count(*) with
count(distinct) gives wrong results when hive.groupby.skewindata=true
(Wangmeng via Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10971 : count(*) with count(distinct) gives wrong results when hive.groupby.skewindata=true (Wangmeng via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19d70748
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19d70748
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19d70748
Branch: refs/heads/spark
Commit: 19d70748fc5e897a58124e2fb8c938b99cdb101b
Parents: 330c31a
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Wed Jun 10 00:06:31 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jun 10 00:07:32 2015 -0700
----------------------------------------------------------------------
.../correlation/ReduceSinkDeDuplication.java | 3 +
.../test/results/clientpositive/groupby10.q.out | 140 ++++++++++++++++---
.../test/results/clientpositive/groupby11.q.out | 70 ++++++++--
.../test/results/clientpositive/groupby2.q.out | 33 ++++-
.../clientpositive/groupby2_map_skew.q.out | 33 ++++-
.../test/results/clientpositive/groupby8.q.out | 140 ++++++++++++++++---
.../clientpositive/groupby8_map_skew.q.out | 70 ++++++++--
.../results/clientpositive/groupby_cube1.q.out | 29 +++-
.../clientpositive/groupby_rollup1.q.out | 29 +++-
.../clientpositive/spark/groupby10.q.out | 88 ++++++++++--
.../clientpositive/spark/groupby11.q.out | 44 +++++-
.../results/clientpositive/spark/groupby2.q.out | 17 ++-
.../results/clientpositive/spark/groupby8.q.out | 88 ++++++++++--
.../spark/groupby8_map_skew.q.out | 44 +++++-
.../clientpositive/spark/groupby_cube1.q.out | 17 ++-
.../clientpositive/spark/groupby_rollup1.q.out | 17 ++-
.../results/clientpositive/tez/groupby2.q.out | 17 ++-
17 files changed, 761 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
index 8ac9ca7..7b5f9b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
@@ -595,6 +595,9 @@ public class ReduceSinkDeDuplication implements Transform {
CorrelationUtilities.findPossibleParent(
start, ReduceSinkOperator.class, dedupCtx.trustScript());
if (pRS != null && merge(cRS, pRS, dedupCtx.minReducer())) {
+ if (dedupCtx.getPctx().getConf().getBoolVar(HiveConf.ConfVars.HIVEGROUPBYSKEW)) {
+ return false;
+ }
CorrelationUtilities.removeReduceSinkForGroupBy(cRS, cGBY, dedupCtx.getPctx(), dedupCtx);
pRS.getConf().setDeduplicated(true);
return true;
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby10.q.out b/ql/src/test/results/clientpositive/groupby10.q.out
index 15969ad..5297107 100644
--- a/ql/src/test/results/clientpositive/groupby10.q.out
+++ b/ql/src/test/results/clientpositive/groupby10.q.out
@@ -46,11 +46,13 @@ INSERT OVERWRITE TABLE dest2 SELECT INPUT.key, sum(substr(INPUT.value,5)), sum(d
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -82,7 +84,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -108,10 +134,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -124,7 +150,31 @@ STAGE PLANS:
Group By Operator
aggregations: sum(KEY._col1:0._col0), sum(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -150,7 +200,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM INPUT
@@ -241,11 +291,13 @@ INSERT OVERWRITE TABLE dest2 SELECT INPUT.key, sum(substr(INPUT.value,5)), sum(d
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -277,7 +329,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -303,10 +379,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -319,7 +395,31 @@ STAGE PLANS:
Group By Operator
aggregations: sum(KEY._col1:0._col0), sum(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -345,7 +445,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM INPUT
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby11.q.out b/ql/src/test/results/clientpositive/groupby11.q.out
index c55ebbf..86568e8 100644
--- a/ql/src/test/results/clientpositive/groupby11.q.out
+++ b/ql/src/test/results/clientpositive/groupby11.q.out
@@ -34,11 +34,13 @@ INSERT OVERWRITE TABLE dest2 partition(ds='111')
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -70,7 +72,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -98,10 +124,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -114,7 +140,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -142,7 +192,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby2.q.out b/ql/src/test/results/clientpositive/groupby2.q.out
index 37a85d5..4ecb807 100644
--- a/ql/src/test/results/clientpositive/groupby2.q.out
+++ b/ql/src/test/results/clientpositive/groupby2.q.out
@@ -16,8 +16,9 @@ INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -39,7 +40,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0), sum(KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: double)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -65,7 +90,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest_g2
- Stage: Stage-2
+ Stage: Stage-3
Stats-Aggr Operator
PREHOOK: query: FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby2_map_skew.q.out b/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
index 9ca1819..813ae5c 100644
--- a/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
+++ b/ql/src/test/results/clientpositive/groupby2_map_skew.q.out
@@ -16,8 +16,9 @@ INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(s
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -46,7 +47,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0), sum(VALUE._col1)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: double)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -72,7 +97,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-2
+ Stage: Stage-3
Stats-Aggr Operator
PREHOOK: query: FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby8.q.out b/ql/src/test/results/clientpositive/groupby8.q.out
index ea34021..0b13817 100644
--- a/ql/src/test/results/clientpositive/groupby8.q.out
+++ b/ql/src/test/results/clientpositive/groupby8.q.out
@@ -30,11 +30,13 @@ INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5))
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -66,7 +68,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -92,10 +118,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -108,7 +134,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -134,7 +184,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM SRC
@@ -801,11 +851,13 @@ INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5))
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -837,7 +889,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -863,10 +939,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -879,7 +955,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -905,7 +1005,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM SRC
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby8_map_skew.q.out b/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
index 5160158..3ab8a4c 100644
--- a/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
+++ b/ql/src/test/results/clientpositive/groupby8_map_skew.q.out
@@ -30,11 +30,13 @@ INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5))
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
- Stage-4 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-0
+ Stage-5 depends on stages: Stage-2
+ Stage-6 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -78,7 +80,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -104,10 +130,10 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -120,7 +146,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -146,7 +196,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest2
- Stage: Stage-5
+ Stage: Stage-7
Stats-Aggr Operator
PREHOOK: query: FROM SRC
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby_cube1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_cube1.q.out b/ql/src/test/results/clientpositive/groupby_cube1.q.out
index dc36c7a..62e9c54 100644
--- a/ql/src/test/results/clientpositive/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/groupby_cube1.q.out
@@ -377,7 +377,8 @@ SELECT key, count(distinct val) FROM T1 GROUP BY key with cube
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -405,7 +406,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: final
outputColumnNames: _col0, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
pruneGroupingSetId: true
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/groupby_rollup1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/groupby_rollup1.q.out
index bc7fed3..8e04d86 100644
--- a/ql/src/test/results/clientpositive/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/groupby_rollup1.q.out
@@ -281,7 +281,8 @@ SELECT key, count(distinct val) FROM T1 GROUP BY key with rollup
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -309,7 +310,31 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: final
outputColumnNames: _col0, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
pruneGroupingSetId: true
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out
index 9d3cf36..dd9d9fe 100644
--- a/ql/src/test/results/clientpositive/spark/groupby10.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out
@@ -55,11 +55,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 5 <- Reducer 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: input
@@ -73,7 +75,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: int)
Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: input
@@ -92,7 +94,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -107,12 +123,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(KEY._col1:0._col0), sum(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -251,11 +281,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 5 <- Reducer 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: input
@@ -269,7 +301,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: int)
Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: input
@@ -288,7 +320,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -303,12 +349,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(KEY._col1:0._col0), sum(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: int)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE
Select Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby11.q.out b/ql/src/test/results/clientpositive/spark/groupby11.q.out
index ccacda4..a0d0139 100644
--- a/ql/src/test/results/clientpositive/spark/groupby11.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby11.q.out
@@ -43,11 +43,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 5 <- Reducer 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: src
@@ -61,7 +63,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: value (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: src
@@ -80,7 +82,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -95,12 +111,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(KEY._col1:0._col0), count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby2.q.out b/ql/src/test/results/clientpositive/spark/groupby2.q.out
index f9e3459..cf01d51 100644
--- a/ql/src/test/results/clientpositive/spark/groupby2.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby2.q.out
@@ -24,6 +24,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -45,7 +46,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0), sum(KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: double)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out
index 307395f..c249b61 100644
--- a/ql/src/test/results/clientpositive/spark/groupby8.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out
@@ -39,11 +39,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 5 <- Reducer 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: src
@@ -57,7 +59,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: src
@@ -76,7 +78,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -91,12 +107,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -811,11 +841,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
+ Reducer 5 <- Reducer 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: src
@@ -829,7 +861,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: src
@@ -848,7 +880,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -863,12 +909,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out b/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out
index ba04a57..2fb1d73 100644
--- a/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out
@@ -39,11 +39,13 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 31)
- Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 31)
+ Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 31)
+ Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 31)
+ Reducer 3 <- Reducer 2 (GROUP, 31)
+ Reducer 5 <- Reducer 4 (GROUP, 31)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 6
Map Operator Tree:
TableScan
alias: src
@@ -63,7 +65,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Map 7
Map Operator Tree:
TableScan
alias: src
@@ -88,7 +90,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -103,12 +119,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
index 21c88b1..25bfbec 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
@@ -398,6 +398,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -425,7 +426,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: final
outputColumnNames: _col0, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
pruneGroupingSetId: true
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
index 92b6d19..0a06c65 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
@@ -296,6 +296,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -323,7 +324,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: complete
+ mode: partials
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: final
outputColumnNames: _col0, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
pruneGroupingSetId: true
http://git-wip-us.apache.org/repos/asf/hive/blob/19d70748/ql/src/test/results/clientpositive/tez/groupby2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/groupby2.q.out b/ql/src/test/results/clientpositive/tez/groupby2.q.out
index bb04ed2..dcf6806 100644
--- a/ql/src/test/results/clientpositive/tez/groupby2.q.out
+++ b/ql/src/test/results/clientpositive/tez/groupby2.q.out
@@ -25,6 +25,7 @@ STAGE PLANS:
Tez
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -46,7 +47,21 @@ STAGE PLANS:
Group By Operator
aggregations: count(DISTINCT KEY._col1:0._col0), sum(KEY._col1:0._col0)
keys: KEY._col0 (type: string)
- mode: complete
+ mode: partial1
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: double)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
[23/61] [abbrv] hive git commit: HIVE-10841: [WHERE col is not null]
does not work sometimes for queries with many JOIN statements (Laljo John
Pullokkaran reviewd by Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10841: [WHERE col is not null] does not work sometimes for queries with many JOIN statements (Laljo John Pullokkaran reviewd by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/97802f7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/97802f7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/97802f7d
Branch: refs/heads/spark
Commit: 97802f7d84558eeb7bd473169f562527ab3107f7
Parents: e236314
Author: jpullokk <jp...@apache.org>
Authored: Fri Jun 12 12:38:10 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Fri Jun 12 12:38:10 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 25 ++
ql/src/test/queries/clientpositive/join42.q | 36 +++
ql/src/test/results/clientpositive/join42.q.out | 312 +++++++++++++++++++
3 files changed, 373 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/97802f7d/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 428ba38..58ee605 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -8493,12 +8493,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if (tree.getJoinSrc() == null) {
return;
}
+
// make array with QBJoinTree : outer most(0) --> inner most(n)
List<QBJoinTree> trees = new ArrayList<QBJoinTree>();
for (;tree != null; tree = tree.getJoinSrc()) {
trees.add(tree);
}
+
// merging from 'target'(inner) to 'node'(outer)
+ boolean mergedQBJTree = false;
for (int i = trees.size() - 1; i >= 0; i--) {
QBJoinTree target = trees.get(i);
if (target == null) {
@@ -8528,6 +8531,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
mergeJoins(qb, node, target, pos, mergeDetails.getSecond());
trees.set(j, null);
+ mergedQBJTree = true;
continue; // continue merging with next alias
}
/*
@@ -8540,6 +8544,27 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
}
+
+ // Now that we reordered QBJoinTrees, update leftaliases of all
+ // QBJoinTree from innermost to outer
+ if ((trees.size() > 1) && mergedQBJTree) {
+ QBJoinTree curQBJTree = null;
+ QBJoinTree prevQBJTree = null;
+ for (int i = trees.size() - 1; i >= 0; i--) {
+ curQBJTree = trees.get(i);
+ if (curQBJTree != null) {
+ if (prevQBJTree != null) {
+ ArrayList<String> newCurLeftAliases = new ArrayList<String>();
+ newCurLeftAliases.addAll(Arrays.asList(prevQBJTree.getLeftAliases()));
+ newCurLeftAliases.addAll(Arrays.asList(prevQBJTree.getRightAliases()));
+ curQBJTree
+ .setLeftAliases(newCurLeftAliases.toArray(new String[newCurLeftAliases.size()]));
+ }
+ prevQBJTree = curQBJTree;
+ }
+ }
+ }
+
// reconstruct join tree
QBJoinTree current = null;
for (int i = 0; i < trees.size(); i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/97802f7d/ql/src/test/queries/clientpositive/join42.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join42.q b/ql/src/test/queries/clientpositive/join42.q
new file mode 100644
index 0000000..f379058
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join42.q
@@ -0,0 +1,36 @@
+create table L as select 4436 id;
+create table LA as select 4436 loan_id, 4748 aid, 4415 pi_id;
+create table FR as select 4436 loan_id;
+create table A as select 4748 id;
+create table PI as select 4415 id;
+
+create table acct as select 4748 aid, 10 acc_n, 122 brn;
+insert into table acct values(4748, null, null);
+insert into table acct values(4748, null, null);
+
+--[HIVE-10841] (WHERE col is not null) does not work sometimes for queries with many JOIN statements
+explain select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null;
+
+select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null;
http://git-wip-us.apache.org/repos/asf/hive/blob/97802f7d/ql/src/test/results/clientpositive/join42.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join42.q.out b/ql/src/test/results/clientpositive/join42.q.out
new file mode 100644
index 0000000..9fdf673
--- /dev/null
+++ b/ql/src/test/results/clientpositive/join42.q.out
@@ -0,0 +1,312 @@
+PREHOOK: query: create table L as select 4436 id
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@L
+POSTHOOK: query: create table L as select 4436 id
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@L
+PREHOOK: query: create table LA as select 4436 loan_id, 4748 aid, 4415 pi_id
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@LA
+POSTHOOK: query: create table LA as select 4436 loan_id, 4748 aid, 4415 pi_id
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@LA
+PREHOOK: query: create table FR as select 4436 loan_id
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@FR
+POSTHOOK: query: create table FR as select 4436 loan_id
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@FR
+PREHOOK: query: create table A as select 4748 id
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@A
+POSTHOOK: query: create table A as select 4748 id
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@A
+PREHOOK: query: create table PI as select 4415 id
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@PI
+POSTHOOK: query: create table PI as select 4415 id
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@PI
+PREHOOK: query: create table acct as select 4748 aid, 10 acc_n, 122 brn
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acct
+POSTHOOK: query: create table acct as select 4748 aid, 10 acc_n, 122 brn
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acct
+PREHOOK: query: insert into table acct values(4748, null, null)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@acct
+POSTHOOK: query: insert into table acct values(4748, null, null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@acct
+POSTHOOK: Lineage: acct.acc_n EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: acct.aid EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: acct.brn EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: insert into table acct values(4748, null, null)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@acct
+POSTHOOK: query: insert into table acct values(4748, null, null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@acct
+POSTHOOK: Lineage: acct.acc_n EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: acct.aid EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: acct.brn EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: --[HIVE-10841] (WHERE col is not null) does not work sometimes for queries with many JOIN statements
+explain select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null
+PREHOOK: type: QUERY
+POSTHOOK: query: --[HIVE-10841] (WHERE col is not null) does not work sometimes for queries with many JOIN statements
+explain select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: l
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (id = 4436) (type: boolean)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 4436 (type: int)
+ sort order: +
+ Map-reduce partition columns: 4436 (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ alias: la
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((loan_id is not null and aid is not null) and pi_id is not null) and (loan_id = 4436)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 4436 (type: int)
+ sort order: +
+ Map-reduce partition columns: 4436 (type: int)
+ Statistics: Num rows: 1 Data size: 14 Basic stats: COMPLETE Column stats: NONE
+ value expressions: aid (type: int), pi_id (type: int)
+ TableScan
+ alias: fr
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (loan_id = 4436) (type: boolean)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 4436 (type: int)
+ sort order: +
+ Map-reduce partition columns: 4436 (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 id (type: int)
+ 1 loan_id (type: int)
+ 2 loan_id (type: int)
+ outputColumnNames: _col5, _col6
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col5 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col5 (type: int)
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col6 (type: int)
+ TableScan
+ alias: a
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: id (type: int)
+ sort order: +
+ Map-reduce partition columns: id (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ alias: acct
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (aid is not null and brn is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: aid (type: int)
+ sort order: +
+ Map-reduce partition columns: aid (type: int)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: acc_n (type: int), brn (type: int)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 2
+ keys:
+ 0 _col5 (type: int)
+ 1 id (type: int)
+ 2 aid (type: int)
+ outputColumnNames: _col6, _col19, _col20
+ Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col6 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col6 (type: int)
+ Statistics: Num rows: 4 Data size: 17 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col19 (type: int), _col20 (type: int)
+ TableScan
+ alias: pi
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: id (type: int)
+ sort order: +
+ Map-reduce partition columns: id (type: int)
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col6 (type: int)
+ 1 id (type: int)
+ outputColumnNames: _col19, _col20
+ Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col19 (type: int), _col20 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 18 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a
+PREHOOK: Input: default@acct
+PREHOOK: Input: default@fr
+PREHOOK: Input: default@l
+PREHOOK: Input: default@la
+PREHOOK: Input: default@pi
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ acct.ACC_N,
+ acct.brn
+FROM L
+JOIN LA ON L.id = LA.loan_id
+JOIN FR ON L.id = FR.loan_id
+JOIN A ON LA.aid = A.id
+JOIN PI ON PI.id = LA.pi_id
+JOIN acct ON A.id = acct.aid
+WHERE
+ L.id = 4436
+ and acct.brn is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a
+POSTHOOK: Input: default@acct
+POSTHOOK: Input: default@fr
+POSTHOOK: Input: default@l
+POSTHOOK: Input: default@la
+POSTHOOK: Input: default@pi
+#### A masked pattern was here ####
+10 122
[24/61] [abbrv] hive git commit: HIVE-10944 : Fix HS2 for Metrics
(Szehon, reviewed by Sergey Shelukhin and Lenni Kuff)
Posted by xu...@apache.org.
HIVE-10944 : Fix HS2 for Metrics (Szehon, reviewed by Sergey Shelukhin and Lenni Kuff)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fd11679
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fd11679
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fd11679
Branch: refs/heads/spark
Commit: 5fd11679a6184ec5abdc9be00b328c310ddfe62c
Parents: 97802f7
Author: Szehon Ho <sz...@cloudera.com>
Authored: Fri Jun 12 16:32:26 2015 -0700
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Fri Jun 12 16:33:38 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/common/JvmPauseMonitor.java | 12 +++--
.../hive/common/metrics/LegacyMetrics.java | 51 ++++----------------
.../hive/common/metrics/common/Metrics.java | 8 +--
.../common/metrics/common/MetricsFactory.java | 30 +++++++++---
.../metrics/metrics2/CodahaleMetrics.java | 41 +++++-----------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../hive/common/metrics/TestLegacyMetrics.java | 6 +--
.../metrics/metrics2/TestCodahaleMetrics.java | 16 +++---
.../hadoop/hive/metastore/HiveMetaStore.java | 10 ++--
.../apache/hive/service/server/HiveServer2.java | 6 +--
10 files changed, 72 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java b/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
index c3949f2..ec5ac4a 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JvmPauseMonitor.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.metrics.common.Metrics;
import org.apache.hadoop.hive.common.metrics.common.MetricsFactory;
import org.apache.hadoop.util.Daemon;
@@ -199,10 +200,13 @@ public class JvmPauseMonitor {
}
private void incrementMetricsCounter(String name, long count) {
- try {
- MetricsFactory.getMetricsInstance().incrementCounter(name, count);
- } catch (Exception e) {
- LOG.warn("Error Reporting JvmPauseMonitor to Metrics system", e);
+ Metrics metrics = MetricsFactory.getInstance();
+ if (metrics != null) {
+ try {
+ metrics.incrementCounter(name, count);
+ } catch (Exception e) {
+ LOG.warn("Error Reporting JvmPauseMonitor to Metrics system", e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
index 14f7afb..e811339 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java
@@ -149,7 +149,6 @@ public class LegacyMetrics implements Metrics {
}
}
-
private static final ThreadLocal<HashMap<String, MetricsScope>> threadLocalScopes
= new ThreadLocal<HashMap<String,MetricsScope>>() {
@Override
@@ -158,31 +157,16 @@ public class LegacyMetrics implements Metrics {
}
};
- private boolean initialized = false;
-
- public void init(HiveConf conf) throws Exception {
- if (!initialized) {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- mbs.registerMBean(metrics, oname);
- initialized = true;
- }
- }
-
- public boolean isInitialized() {
- return initialized;
+ public LegacyMetrics(HiveConf conf) throws Exception {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ mbs.registerMBean(metrics, oname);
}
public Long incrementCounter(String name) throws IOException{
- if (!initialized) {
- return null;
- }
return incrementCounter(name,Long.valueOf(1));
}
public Long incrementCounter(String name, long increment) throws IOException{
- if (!initialized) {
- return null;
- }
Long value;
synchronized(metrics) {
if (!metrics.hasKey(name)) {
@@ -197,23 +181,14 @@ public class LegacyMetrics implements Metrics {
}
public void set(String name, Object value) throws IOException{
- if (!initialized) {
- return;
- }
metrics.put(name,value);
}
public Object get(String name) throws IOException{
- if (!initialized) {
- return null;
- }
return metrics.get(name);
}
public void startScope(String name) throws IOException{
- if (!initialized) {
- return;
- }
if (threadLocalScopes.get().containsKey(name)) {
threadLocalScopes.get().get(name).open();
} else {
@@ -222,9 +197,6 @@ public class LegacyMetrics implements Metrics {
}
public MetricsScope getScope(String name) throws IOException {
- if (!initialized) {
- return null;
- }
if (threadLocalScopes.get().containsKey(name)) {
return threadLocalScopes.get().get(name);
} else {
@@ -233,9 +205,6 @@ public class LegacyMetrics implements Metrics {
}
public void endScope(String name) throws IOException{
- if (!initialized) {
- return;
- }
if (threadLocalScopes.get().containsKey(name)) {
threadLocalScopes.get().get(name).close();
}
@@ -247,16 +216,14 @@ public class LegacyMetrics implements Metrics {
*
* Note that threadLocalScopes ThreadLocal is *not* cleared in this call.
*/
- public void deInit() throws Exception {
+ public void close() throws Exception {
synchronized (metrics) {
- if (initialized) {
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- if (mbs.isRegistered(oname)) {
- mbs.unregisterMBean(oname);
- }
- metrics.clear();
- initialized = false;
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ if (mbs.isRegistered(oname)) {
+ mbs.unregisterMBean(oname);
}
+ metrics.clear();
+ threadLocalScopes.remove();
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
index 13a5336..27b69cc 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java
@@ -28,16 +28,12 @@ import java.io.IOException;
*/
public interface Metrics {
- /**
- * Initialize Metrics system with given Hive configuration.
- * @param conf
- */
- public void init(HiveConf conf) throws Exception;
+ //Must declare CTOR taking in HiveConf.
/**
* Deinitializes the Metrics system.
*/
- public void deInit() throws Exception;
+ public void close() throws Exception;
/**
* @param name
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
index 12a309d..8769d68 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java
@@ -20,29 +20,43 @@ package org.apache.hadoop.hive.common.metrics.common;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.ReflectionUtils;
+import java.lang.reflect.Constructor;
+
/**
* Class that manages a static Metric instance for this process.
*/
public class MetricsFactory {
- private static Metrics metrics;
- private static Object initLock = new Object();
+ //Volatile ensures that static access returns Metrics instance in fully-initialized state.
+ //Alternative is to synchronize static access, which has performance penalties.
+ private volatile static Metrics metrics;
+ /**
+ * Initializes static Metrics instance.
+ */
public synchronized static void init(HiveConf conf) throws Exception {
if (metrics == null) {
- metrics = (Metrics) ReflectionUtils.newInstance(conf.getClassByName(
- conf.getVar(HiveConf.ConfVars.HIVE_METRICS_CLASS)), conf);
+ Class metricsClass = conf.getClassByName(
+ conf.getVar(HiveConf.ConfVars.HIVE_METRICS_CLASS));
+ Constructor constructor = metricsClass.getConstructor(HiveConf.class);
+ metrics = (Metrics) constructor.newInstance(conf);
}
- metrics.init(conf);
}
- public synchronized static Metrics getMetricsInstance() {
+ /**
+ * Returns static Metrics instance, null if not initialized or closed.
+ */
+ public static Metrics getInstance() {
return metrics;
}
- public synchronized static void deInit() throws Exception {
+ /**
+ * Closes and removes static Metrics instance.
+ */
+ public synchronized static void close() throws Exception {
if (metrics != null) {
- metrics.deInit();
+ metrics.close();
+ metrics = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
index e59da99..ae353d0 100644
--- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
+++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java
@@ -77,7 +77,6 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
private LoadingCache<String, Timer> timers;
private LoadingCache<String, Counter> counters;
- private boolean initialized = false;
private HiveConf conf;
private final Set<Closeable> reporters = new HashSet<Closeable>();
@@ -139,11 +138,7 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
}
}
- public synchronized void init(HiveConf conf) throws Exception {
- if (initialized) {
- return;
- }
-
+ public CodahaleMetrics(HiveConf conf) throws Exception {
this.conf = conf;
//Codahale artifacts are lazily-created.
timers = CacheBuilder.newBuilder().build(
@@ -190,32 +185,23 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
}
}
initReporting(finalReporterList);
- initialized = true;
}
- public synchronized void deInit() throws Exception {
- if (initialized) {
- if (reporters != null) {
- for (Closeable reporter : reporters) {
- reporter.close();
- }
+ public void close() throws Exception {
+ if (reporters != null) {
+ for (Closeable reporter : reporters) {
+ reporter.close();
}
- for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) {
- metricRegistry.remove(metric.getKey());
- }
- timers.invalidateAll();
- counters.invalidateAll();
- initialized = false;
}
+ for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) {
+ metricRegistry.remove(metric.getKey());
+ }
+ timers.invalidateAll();
+ counters.invalidateAll();
}
public void startScope(String name) throws IOException {
- synchronized (this) {
- if (!initialized) {
- return;
- }
- }
name = API_PREFIX + name;
if (threadLocalScopes.get().containsKey(name)) {
threadLocalScopes.get().get(name).open();
@@ -224,12 +210,7 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co
}
}
- public void endScope(String name) throws IOException{
- synchronized (this) {
- if (!initialized) {
- return;
- }
- }
+ public void endScope(String name) throws IOException {
name = API_PREFIX + name;
if (threadLocalScopes.get().containsKey(name)) {
threadLocalScopes.get().get(name).close();
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4e36bae..27f68df 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1725,7 +1725,7 @@ public class HiveConf extends Configuration {
"Hive metrics subsystem implementation class."),
HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX",
"Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, comma separated list of JMX, CONSOLE, JSON_FILE"),
- HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "file:///tmp/my-logging.properties",
+ HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "file:///tmp/report.json",
"For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of JSON metrics file. " +
"This file will get overwritten at every interval."),
HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5s",
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
index c14c7ee..c3e8282 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/TestLegacyMetrics.java
@@ -47,16 +47,16 @@ public class TestLegacyMetrics {
@Before
public void before() throws Exception {
- MetricsFactory.deInit();
+ MetricsFactory.close();
HiveConf conf = new HiveConf();
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, LegacyMetrics.class.getCanonicalName());
MetricsFactory.init(conf);
- metrics = (LegacyMetrics) MetricsFactory.getMetricsInstance();
+ metrics = (LegacyMetrics) MetricsFactory.getInstance();
}
@After
public void after() throws Exception {
- MetricsFactory.deInit();
+ MetricsFactory.close();
}
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
index 8749349..954b388 100644
--- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
+++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java
@@ -63,20 +63,20 @@ public class TestCodahaleMetrics {
conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms");
MetricsFactory.init(conf);
- metricRegistry = ((CodahaleMetrics) MetricsFactory.getMetricsInstance()).getMetricRegistry();
+ metricRegistry = ((CodahaleMetrics) MetricsFactory.getInstance()).getMetricRegistry();
}
@After
public void after() throws Exception {
- MetricsFactory.deInit();
+ MetricsFactory.close();
}
@Test
public void testScope() throws Exception {
int runs = 5;
for (int i = 0; i < runs; i++) {
- MetricsFactory.getMetricsInstance().startScope("method1");
- MetricsFactory.getMetricsInstance().endScope("method1");
+ MetricsFactory.getInstance().startScope("method1");
+ MetricsFactory.getInstance().endScope("method1");
}
Timer timer = metricRegistry.getTimers().get("api_method1");
@@ -89,7 +89,7 @@ public class TestCodahaleMetrics {
public void testCount() throws Exception {
int runs = 5;
for (int i = 0; i < runs; i++) {
- MetricsFactory.getMetricsInstance().incrementCounter("count1");
+ MetricsFactory.getInstance().incrementCounter("count1");
}
Counter counter = metricRegistry.getCounters().get("count1");
Assert.assertEquals(5L, counter.getCount());
@@ -104,8 +104,8 @@ public class TestCodahaleMetrics {
executorService.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
- MetricsFactory.getMetricsInstance().startScope("method2");
- MetricsFactory.getMetricsInstance().endScope("method2");
+ MetricsFactory.getInstance().startScope("method2");
+ MetricsFactory.getInstance().endScope("method2");
return null;
}
});
@@ -121,7 +121,7 @@ public class TestCodahaleMetrics {
public void testFileReporting() throws Exception {
int runs = 5;
for (int i = 0; i < runs; i++) {
- MetricsFactory.getMetricsInstance().incrementCounter("count2");
+ MetricsFactory.getInstance().incrementCounter("count2");
Thread.sleep(100);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 914f2e7..ae9c119 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -748,9 +748,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
incrementCounter(function);
logInfo((getIpAddress() == null ? "" : "source:" + getIpAddress() + " ") +
function + extraLogInfo);
- if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ if (MetricsFactory.getInstance() != null) {
try {
- MetricsFactory.getMetricsInstance().startScope(function);
+ MetricsFactory.getInstance().startScope(function);
} catch (IOException e) {
LOG.debug("Exception when starting metrics scope"
+ e.getClass().getName() + " " + e.getMessage(), e);
@@ -792,9 +792,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
private void endFunction(String function, MetaStoreEndFunctionContext context) {
- if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
+ if (MetricsFactory.getInstance() != null) {
try {
- MetricsFactory.getMetricsInstance().endScope(function);
+ MetricsFactory.getInstance().endScope(function);
} catch (IOException e) {
LOG.debug("Exception when closing metrics scope" + e);
}
@@ -823,7 +823,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
try {
- MetricsFactory.deInit();
+ MetricsFactory.close();
} catch (Exception e) {
LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
+ e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/hive/blob/5fd11679/service/src/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 0959cba..00ab75f 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -308,9 +308,9 @@ public class HiveServer2 extends CompositeService {
HiveConf hiveConf = this.getHiveConf();
super.stop();
// Shutdown Metrics
- if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
+ if (MetricsFactory.getInstance() != null) {
try {
- MetricsFactory.getMetricsInstance().deInit();
+ MetricsFactory.close();
} catch (Exception e) {
LOG.error("error in Metrics deinit: " + e.getClass().getName() + " "
+ e.getMessage(), e);
@@ -359,7 +359,7 @@ public class HiveServer2 extends CompositeService {
server.start();
if (hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) {
- MetricsFactory.getMetricsInstance().init(hiveConf);
+ MetricsFactory.init(hiveConf);
}
try {
JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(hiveConf);
[16/61] [abbrv] hive git commit: HIVE-10968 : Windows: analyze json
table via beeline failed throwing Class
org.apache.hive.hcatalog.data.JsonSerDe not found (Hari Subramaniyan,
reviewed by Thejas Nair)
Posted by xu...@apache.org.
HIVE-10968 : Windows: analyze json table via beeline failed throwing Class org.apache.hive.hcatalog.data.JsonSerDe not found (Hari Subramaniyan, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/40f348f6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/40f348f6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/40f348f6
Branch: refs/heads/spark
Commit: 40f348f6bb9d660d7f3f88c67c96d576fc6638e9
Parents: f2600e9
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Wed Jun 10 14:23:42 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Wed Jun 10 14:23:42 2015 -0700
----------------------------------------------------------------------
bin/ext/hiveserver2.cmd | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/40f348f6/bin/ext/hiveserver2.cmd
----------------------------------------------------------------------
diff --git a/bin/ext/hiveserver2.cmd b/bin/ext/hiveserver2.cmd
index 86a35e0..5431de6 100644
--- a/bin/ext/hiveserver2.cmd
+++ b/bin/ext/hiveserver2.cmd
@@ -76,7 +76,7 @@ goto :EOF
@echo ^<name^>HiveServer2^</name^>
@echo ^<description^>Hadoop HiveServer2 Service^</description^>
@echo ^<executable^>%JAVA_HOME%\bin\java^</executable^>
-@echo ^<arguments^>%JAVA_HEAP_MAX% -XX:MaxPermSize=512m %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^</arguments^>
+@echo ^<arguments^>%JAVA_HEAP_MAX% -XX:MaxPermSize=512m %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH%;%HIVE_HOME%\hcatalog\share\hcatalog\* %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^</arguments^>
@echo ^</service^>
goto :EOF
[54/61] [abbrv] hive git commit: HIVE-11028: Tez: table self join and
join with another table fails with IndexOutOfBoundsException (Jason Dere,
reviewed by John Pullokkaran)
Posted by xu...@apache.org.
HIVE-11028: Tez: table self join and join with another table fails with IndexOutOfBoundsException (Jason Dere, reviewed by John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2dd54ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2dd54ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2dd54ab
Branch: refs/heads/spark
Commit: e2dd54ab180b577b08cf6b0e69310ac81fc99fd3
Parents: 56cffb2
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Jun 19 15:19:09 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Fri Jun 19 15:19:09 2015 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/ConstantPropagate.java | 12 +-
.../ql/optimizer/ConstantPropagateProcCtx.java | 21 ++
.../optimizer/ConstantPropagateProcFactory.java | 69 ++++++-
.../hadoop/hive/ql/parse/TezCompiler.java | 5 +-
.../test/queries/clientpositive/tez_self_join.q | 32 +++
.../clientpositive/tez/explainuser_2.q.out | 4 +-
.../clientpositive/tez/tez_self_join.q.out | 205 +++++++++++++++++++
8 files changed, 343 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index b9f39fb..7b7559a 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -334,6 +334,7 @@ minitez.query.files=bucket_map_join_tez1.q,\
tez_join_tests.q,\
tez_joins_explain.q,\
tez_schema_evolution.q,\
+ tez_self_join.q,\
tez_union.q,\
tez_union2.q,\
tez_union_dynamic_partition.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
index 0027960..b5ee4ef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -62,8 +63,15 @@ public class ConstantPropagate implements Transform {
private static final Log LOG = LogFactory.getLog(ConstantPropagate.class);
protected ParseContext pGraphContext;
+ private ConstantPropagateOption constantPropagateOption;
- public ConstantPropagate() {}
+ public ConstantPropagate() {
+ this(ConstantPropagateOption.FULL);
+ }
+
+ public ConstantPropagate(ConstantPropagateOption option) {
+ this.constantPropagateOption = option;
+ }
/**
* Transform the query tree.
@@ -76,7 +84,7 @@ public class ConstantPropagate implements Transform {
pGraphContext = pactx;
// generate pruned column list for all relevant operators
- ConstantPropagateProcCtx cppCtx = new ConstantPropagateProcCtx();
+ ConstantPropagateProcCtx cppCtx = new ConstantPropagateProcCtx(constantPropagateOption);
// create a walker which walks the tree in a DFS manner while maintaining
// the operator stack. The dispatcher
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
index 6bb2a09..f30e330 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
@@ -43,16 +43,28 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
*/
public class ConstantPropagateProcCtx implements NodeProcessorCtx {
+ public enum ConstantPropagateOption {
+ FULL, // Do full constant propagation
+ SHORTCUT, // Only perform expression short-cutting - remove unnecessary AND/OR operators
+ // if one of the child conditions is true/false.
+ };
+
private static final org.apache.commons.logging.Log LOG = LogFactory
.getLog(ConstantPropagateProcCtx.class);
private final Map<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>> opToConstantExprs;
private final List<Operator<? extends Serializable>> opToDelete;
+ private ConstantPropagateOption constantPropagateOption = ConstantPropagateOption.FULL;
public ConstantPropagateProcCtx() {
+ this(ConstantPropagateOption.FULL);
+ }
+
+ public ConstantPropagateProcCtx(ConstantPropagateOption option) {
opToConstantExprs =
new HashMap<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>>();
opToDelete = new ArrayList<Operator<? extends Serializable>>();
+ this.constantPropagateOption = option;
}
public Map<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>> getOpToConstantExprs() {
@@ -184,4 +196,13 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx {
public List<Operator<? extends Serializable>> getOpToDelete() {
return opToDelete;
}
+
+ public ConstantPropagateOption getConstantPropagateOption() {
+ return constantPropagateOption;
+ }
+
+ public void setConstantPropagateOption(
+ ConstantPropagateOption constantPropagateOption) {
+ this.constantPropagateOption = constantPropagateOption;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index 4a4814d..f9df8e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -193,6 +194,72 @@ public final class ConstantPropagateProcFactory {
}
return evaluateFunction(funcDesc.getGenericUDF(),funcDesc.getChildren(), funcDesc.getChildren());
}
+
+ /**
+ * Fold input expression desc.
+ *
+ * @param desc folding expression
+ * @param constants current propagated constant map
+ * @param cppCtx
+ * @param op processing operator
+ * @param propagate if true, assignment expressions will be added to constants.
+ * @return fold expression
+ * @throws UDFArgumentException
+ */
+ private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
+ ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
+ boolean propagate) throws UDFArgumentException {
+ if (cppCtx.getConstantPropagateOption() == ConstantPropagateOption.SHORTCUT) {
+ return foldExprShortcut(desc, constants, cppCtx, op, tag, propagate);
+ }
+ return foldExprFull(desc, constants, cppCtx, op, tag, propagate);
+ }
+
+ /**
+ * Fold input expression desc, only performing short-cutting.
+ *
+ * Unnecessary AND/OR operations involving a constant true/false value will be eliminated.
+ *
+ * @param desc folding expression
+ * @param constants current propagated constant map
+ * @param cppCtx
+ * @param op processing operator
+ * @param propagate if true, assignment expressions will be added to constants.
+ * @return fold expression
+ * @throws UDFArgumentException
+ */
+ private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
+ ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
+ boolean propagate) throws UDFArgumentException {
+ if (desc instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
+
+ GenericUDF udf = funcDesc.getGenericUDF();
+
+ boolean propagateNext = propagate && propagatableUdfs.contains(udf.getClass());
+ List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>();
+ for (ExprNodeDesc childExpr : desc.getChildren()) {
+ newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
+ }
+
+ // Don't evalulate nondeterministic function since the value can only calculate during runtime.
+ if (!isDeterministicUdf(udf)) {
+ LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulating immediately.");
+ ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
+ return desc;
+ }
+
+ // Check if the function can be short cut.
+ ExprNodeDesc shortcut = shortcutFunction(udf, newExprs, op);
+ if (shortcut != null) {
+ LOG.debug("Folding expression:" + desc + " -> " + shortcut);
+ return shortcut;
+ }
+ ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
+ }
+ return desc;
+ }
+
/**
* Fold input expression desc.
*
@@ -211,7 +278,7 @@ public final class ConstantPropagateProcFactory {
* @return fold expression
* @throws UDFArgumentException
*/
- private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
+ private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
if (desc instanceof ExprNodeGenericFuncDesc) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 56707af..03d41f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption;
import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin;
import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc;
@@ -304,8 +305,10 @@ public class TezCompiler extends TaskCompiler {
// need a new run of the constant folding because we might have created lots
// of "and true and true" conditions.
+ // Rather than run the full constant folding just need to shortcut AND/OR expressions
+ // involving constant true/false values.
if(procCtx.conf.getBoolVar(ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
- new ConstantPropagate().transform(procCtx.parseContext);
+ new ConstantPropagate(ConstantPropagateOption.SHORTCUT).transform(procCtx.parseContext);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/test/queries/clientpositive/tez_self_join.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_self_join.q b/ql/src/test/queries/clientpositive/tez_self_join.q
new file mode 100644
index 0000000..a1ef585
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_self_join.q
@@ -0,0 +1,32 @@
+drop table if exists tez_self_join1;
+drop table if exists tez_self_join2;
+
+create table tez_self_join1(id1 int, id2 string, id3 string);
+insert into table tez_self_join1 values(1, 'aa','bb'), (2, 'ab','ab'), (3,'ba','ba');
+
+create table tez_self_join2(id1 int);
+insert into table tez_self_join2 values(1),(2),(3);
+
+explain
+select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab';
+
+select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab';
+
+drop table tez_self_join1;
+drop table tez_self_join2;
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
index 222e89e..8768198 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out
@@ -239,7 +239,7 @@ Stage-0
| outputColumnNames:["_col0"]
| Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
| Filter Operator [FIL_26]
- | predicate:value is not null (type: boolean)
+ | predicate:((11.0 = 11.0) and value is not null) (type: boolean)
| Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
| TableScan [TS_2]
| alias:z
@@ -1442,7 +1442,7 @@ Stage-0
outputColumnNames:["_col0"]
Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Filter Operator [FIL_26]
- predicate:value is not null (type: boolean)
+ predicate:((11.0 = 11.0) and value is not null) (type: boolean)
Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
TableScan [TS_2]
alias:z
http://git-wip-us.apache.org/repos/asf/hive/blob/e2dd54ab/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_self_join.q.out b/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
new file mode 100644
index 0000000..be538ac
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/tez_self_join.q.out
@@ -0,0 +1,205 @@
+PREHOOK: query: drop table if exists tez_self_join1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tez_self_join1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists tez_self_join2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists tez_self_join2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tez_self_join1(id1 int, id2 string, id3 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tez_self_join1
+POSTHOOK: query: create table tez_self_join1(id1 int, id2 string, id3 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tez_self_join1
+PREHOOK: query: insert into table tez_self_join1 values(1, 'aa','bb'), (2, 'ab','ab'), (3,'ba','ba')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@tez_self_join1
+POSTHOOK: query: insert into table tez_self_join1 values(1, 'aa','bb'), (2, 'ab','ab'), (3,'ba','ba')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@tez_self_join1
+POSTHOOK: Lineage: tez_self_join1.id1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: tez_self_join1.id2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: tez_self_join1.id3 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: create table tez_self_join2(id1 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tez_self_join2
+POSTHOOK: query: create table tez_self_join2(id1 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tez_self_join2
+PREHOOK: query: insert into table tez_self_join2 values(1),(2),(3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@tez_self_join2
+POSTHOOK: query: insert into table tez_self_join2 values(1),(2),(3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@tez_self_join2
+POSTHOOK: Lineage: tez_self_join2.id1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: explain
+select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: self1
+ Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((id2 is not null and id1 is not null) and (id2 = 'ab')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'ab' (type: string)
+ sort order: +
+ Map-reduce partition columns: 'ab' (type: string)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ value expressions: id1 (type: int), id3 (type: string)
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: self2
+ Statistics: Num rows: 3 Data size: 21 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (id3 = 'ab') (type: boolean)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'ab' (type: string)
+ sort order: +
+ Map-reduce partition columns: 'ab' (type: string)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: tez_self_join2
+ Statistics: Num rows: 3 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id1 is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: id1 (type: int)
+ sort order: +
+ Map-reduce partition columns: id1 (type: int)
+ Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 'ab' (type: string)
+ 1 'ab' (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string)
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 id1 (type: int)
+ outputColumnNames: _col2
+ Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: 'ab' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 2 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tez_self_join1
+PREHOOK: Input: default@tez_self_join2
+#### A masked pattern was here ####
+POSTHOOK: query: select s.id2, s.id3
+from
+(
+ select self1.id1, self1.id2, self1.id3
+ from tez_self_join1 self1 join tez_self_join1 self2
+ on self1.id2=self2.id3 ) s
+join tez_self_join2
+on s.id1=tez_self_join2.id1
+where s.id2='ab'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tez_self_join1
+POSTHOOK: Input: default@tez_self_join2
+#### A masked pattern was here ####
+ab ab
+PREHOOK: query: drop table tez_self_join1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tez_self_join1
+PREHOOK: Output: default@tez_self_join1
+POSTHOOK: query: drop table tez_self_join1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tez_self_join1
+POSTHOOK: Output: default@tez_self_join1
+PREHOOK: query: drop table tez_self_join2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tez_self_join2
+PREHOOK: Output: default@tez_self_join2
+POSTHOOK: query: drop table tez_self_join2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tez_self_join2
+POSTHOOK: Output: default@tez_self_join2
[52/61] [abbrv] hive git commit: HIVE-10746: Hive 1.2.0+Tez produces
1-byte FileSplits from mapred.TextInputFormat (Gopal V via Gunther H)
Posted by xu...@apache.org.
HIVE-10746: Hive 1.2.0+Tez produces 1-byte FileSplits from mapred.TextInputFormat (Gopal V via Gunther H)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b98a30b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b98a30b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b98a30b1
Branch: refs/heads/spark
Commit: b98a30b1b9172fdc5a74fc272393c5eec4344c74
Parents: ef6f313
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Thu Jun 18 13:27:29 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Thu Jun 18 13:28:02 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/HiveSplitGenerator.java | 24 ++++++++++++++++++++
1 file changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b98a30b1/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index 52d0996..87881b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -25,11 +25,13 @@ import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -72,6 +74,15 @@ public class HiveSplitGenerator extends InputInitializer {
private final MapWork work;
private final SplitGrouper splitGrouper = new SplitGrouper();
+ private static final String MIN_SPLIT_SIZE;
+ @SuppressWarnings("unused")
+ private static final String MAX_SPLIT_SIZE;
+
+ static {
+ final HadoopShims SHIMS = ShimLoader.getHadoopShims();
+ MIN_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMINSPLITSIZE");
+ MAX_SPLIT_SIZE = SHIMS.getHadoopConfNames().get("MAPREDMAXSPLITSIZE");
+ }
public HiveSplitGenerator(InputInitializerContext initializerContext) throws IOException,
SerDeException {
@@ -97,6 +108,7 @@ public class HiveSplitGenerator extends InputInitializer {
}
+ @SuppressWarnings("unchecked")
@Override
public List<Event> initialize() throws Exception {
// Setup the map work for this thread. Pruning modified the work instance to potentially remove
@@ -123,6 +135,18 @@ public class HiveSplitGenerator extends InputInitializer {
int taskResource = getContext().getVertexTaskResource().getMemory();
int availableSlots = totalResource / taskResource;
+ if (conf.getLong(MIN_SPLIT_SIZE, 1) <= 1) {
+ // broken configuration from mapred-default.xml
+ final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
+ final long minGrouping = conf.getLong(
+ TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE,
+ TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT);
+ final long preferredSplitSize = Math.min(blockSize / 2, minGrouping);
+ jobConf.setLong(MIN_SPLIT_SIZE, preferredSplitSize);
+ LOG.info("The preferred split size is " + preferredSplitSize);
+ }
+
// Create the un-grouped splits
float waves =
conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES,
[44/61] [abbrv] hive git commit: HIVE-11027: Hive on tez: Bucket map
joins fail when hashcode goes negative (Prasanth Jayachandran reviewed by
Vikram Dixit)
Posted by xu...@apache.org.
HIVE-11027: Hive on tez: Bucket map joins fail when hashcode goes negative (Prasanth Jayachandran reviewed by Vikram Dixit)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/03c62cec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/03c62cec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/03c62cec
Branch: refs/heads/spark
Commit: 03c62cec17f55146de3e6dfabdeecaa7deee02f7
Parents: 524cd79
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Jun 17 23:03:50 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Jun 17 23:03:50 2015 -0700
----------------------------------------------------------------------
data/files/sortdp.txt | 32 +++
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 7 +-
.../clientpositive/dynpart_sort_opt_bucketing.q | 73 +++++
.../dynpart_sort_opt_bucketing.q.out | 277 +++++++++++++++++++
4 files changed, 384 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/03c62cec/data/files/sortdp.txt
----------------------------------------------------------------------
diff --git a/data/files/sortdp.txt b/data/files/sortdp.txt
new file mode 100644
index 0000000..dc7781d
--- /dev/null
+++ b/data/files/sortdp.txt
@@ -0,0 +1,32 @@
+ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21
+fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21
+fffe069c9bba031c86f1f49ec87bf75d4feb5cc5c7d430282f739ebc4337c673002015-01-21
+fffdc9482d8a88626887d92851951b92e58dba2cfa7e22caeafe98baac4d2a3e042015-01-21
+fffd4906f05ab50dfa41f4ab226e83de981fd8aa4ce0d4f516e7ebf9566a4213\N\N2015-01-21
+fffd37a9ebbef0ed3d3440f7bacd64b823df685c90c827a75a5e9556126a8d0c\N\N2015-01-21
+fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21
+fffce560fc7c69c5185eda181759a62f30f908c55f04291d612039a3ca16357a\N\N2015-01-21
+fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21
+fffc8a89a8406ba3ed651fc9248fe9b2afb1ee672999ef75d2d9c42484435bca\N\N2015-01-21
+fffc502293653c4249cdeb8fad4ba2437006b56d527eae844d18e6d6324e424d002015-01-21
+fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21
+fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21
+fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21
+fffad5028e769fdf811eea954785629c064826a5a008253136dbd4f189334827\N\N2015-01-21
+fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21
+fff876f50b99444aa7ef901383336299c47f035b87d253283f737d88a05edce8\N\N2015-01-21
+fff66e6646a820cf6d39ba467a735db4ffe4df344256bb1a1cb854086e766810\N\N2015-01-21
+fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21
+fff4cabcf8a5f2f8edd27d5d92659ad80e3513d15ac2df7557a82ade5ed80b86\N\N2015-01-21
+fff42e4d4463fa981df990041ba914deaa1d1c1cdcb1547678ab899e34248526012015-01-21
+fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21
+fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21
+fff2de7c5a9929257705996690aa45e5848e57a85eb91f83dfcfa6ac4219ae1b\N\N2015-01-21
+fff201d3ead782d817f222c73cd241df1683cecdb109a5a22b02004b7b228b14\N\N2015-01-21
+fff0fbdf2bf7bfed9ea314bfa7fa14bf179345dde9ea79838f66acd70b943f0b\N\N2015-01-21
+fff06e59d471f90aa055c3089c1d8e380bbb86d5642142ecd0589b544dd44688002015-01-21
+fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21
+fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21
+ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21
+ffef4b7433ee6008e389a1a4121a82b828123864e563a2afe67dcf29e2b71591\N\N2015-01-21
+ffef34be95399a087af57dd713ca3510377625811bb4232473c1ba223239f284002015-01-21
http://git-wip-us.apache.org/repos/asf/hive/blob/03c62cec/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index 859a28f..f1df608 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -424,11 +424,8 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
}
}
- if (buckNum < 0) {
- buckNum = -1 * buckNum;
- }
-
- return buckNum % numBuckets;
+ // similar to hive's default partitioner, refer DefaultHivePartitioner
+ return (buckNum & Integer.MAX_VALUE) % numBuckets;
}
private void populateCachedDistributionKeys(Object row, int index) throws HiveException {
http://git-wip-us.apache.org/repos/asf/hive/blob/03c62cec/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q
new file mode 100644
index 0000000..95ae6e3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_bucketing.q
@@ -0,0 +1,73 @@
+set hive.vectorized.execution.enabled=false;
+
+drop table if exists t1_staging;
+create table t1_staging(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc)
+into 256 buckets stored as textfile;
+
+load data local inpath '../../data/files/sortdp.txt' overwrite into table t1_staging partition (e='epart');
+
+set hive.optimize.sort.dynamic.partition=true;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.enforce.sorting=true;
+set hive.enforce.bucketing=true;
+
+drop table t1;
+
+create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile;
+
+insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging;
+
+select 'bucket_0';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000000_0;
+select 'bucket_2';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000002_0;
+select 'bucket_4';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000004_0;
+select 'bucket_6';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000006_0;
+select 'bucket_8';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000008_0;
+
+set hive.optimize.sort.dynamic.partition=false;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.enforce.sorting=true;
+set hive.enforce.bucketing=true;
+
+-- disable sorted dynamic partition optimization to make sure the results are correct
+drop table t1;
+
+create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile;
+
+insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging;
+
+select 'bucket_0';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000000_0;
+select 'bucket_2';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000002_0;
+select 'bucket_4';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000004_0;
+select 'bucket_6';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000006_0;
+select 'bucket_8';
+dfs -cat ${hiveconf:hive.metastore.warehouse.dir}/t1/e=epart/000008_0;
http://git-wip-us.apache.org/repos/asf/hive/blob/03c62cec/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out b/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out
new file mode 100644
index 0000000..3992db7
--- /dev/null
+++ b/ql/src/test/results/clientpositive/dynpart_sort_opt_bucketing.q.out
@@ -0,0 +1,277 @@
+PREHOOK: query: drop table if exists t1_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists t1_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table t1_staging(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc)
+into 256 buckets stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1_staging
+POSTHOOK: query: create table t1_staging(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc)
+into 256 buckets stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1_staging
+PREHOOK: query: load data local inpath '../../data/files/sortdp.txt' overwrite into table t1_staging partition (e='epart')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t1_staging
+POSTHOOK: query: load data local inpath '../../data/files/sortdp.txt' overwrite into table t1_staging partition (e='epart')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t1_staging
+POSTHOOK: Output: default@t1_staging@e=epart
+PREHOOK: query: drop table t1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table t1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1_staging
+PREHOOK: Input: default@t1_staging@e=epart
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1_staging
+POSTHOOK: Input: default@t1_staging@e=epart
+POSTHOOK: Output: default@t1@e=epart
+POSTHOOK: Lineage: t1 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ]
+PREHOOK: query: select 'bucket_0'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_0
+fffdc9482d8a88626887d92851951b92e58dba2cfa7e22caeafe98baac4d2a3e042015-01-21
+fffd4906f05ab50dfa41f4ab226e83de981fd8aa4ce0d4f516e7ebf9566a4213\N\N2015-01-21
+fffd37a9ebbef0ed3d3440f7bacd64b823df685c90c827a75a5e9556126a8d0c\N\N2015-01-21
+fffce560fc7c69c5185eda181759a62f30f908c55f04291d612039a3ca16357a\N\N2015-01-21
+fffc502293653c4249cdeb8fad4ba2437006b56d527eae844d18e6d6324e424d002015-01-21
+fffad5028e769fdf811eea954785629c064826a5a008253136dbd4f189334827\N\N2015-01-21
+fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21
+fff2de7c5a9929257705996690aa45e5848e57a85eb91f83dfcfa6ac4219ae1b\N\N2015-01-21
+PREHOOK: query: select 'bucket_2'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_2
+fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21
+fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21
+fff876f50b99444aa7ef901383336299c47f035b87d253283f737d88a05edce8\N\N2015-01-21
+fff66e6646a820cf6d39ba467a735db4ffe4df344256bb1a1cb854086e766810\N\N2015-01-21
+fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21
+fff06e59d471f90aa055c3089c1d8e380bbb86d5642142ecd0589b544dd44688002015-01-21
+fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21
+PREHOOK: query: select 'bucket_4'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_4
+fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21
+fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21
+fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21
+fff201d3ead782d817f222c73cd241df1683cecdb109a5a22b02004b7b228b14\N\N2015-01-21
+fff0fbdf2bf7bfed9ea314bfa7fa14bf179345dde9ea79838f66acd70b943f0b\N\N2015-01-21
+PREHOOK: query: select 'bucket_6'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_6'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_6
+ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21
+fffe069c9bba031c86f1f49ec87bf75d4feb5cc5c7d430282f739ebc4337c673002015-01-21
+fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21
+fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21
+fff4cabcf8a5f2f8edd27d5d92659ad80e3513d15ac2df7557a82ade5ed80b86\N\N2015-01-21
+fff42e4d4463fa981df990041ba914deaa1d1c1cdcb1547678ab899e34248526012015-01-21
+ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21
+ffef34be95399a087af57dd713ca3510377625811bb4232473c1ba223239f284002015-01-21
+PREHOOK: query: select 'bucket_8'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_8'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_8
+fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21
+fffc8a89a8406ba3ed651fc9248fe9b2afb1ee672999ef75d2d9c42484435bca\N\N2015-01-21
+fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21
+ffef4b7433ee6008e389a1a4121a82b828123864e563a2afe67dcf29e2b71591\N\N2015-01-21
+PREHOOK: query: -- disable sorted dynamic partition optimization to make sure the results are correct
+drop table t1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t1
+PREHOOK: Output: default@t1
+POSTHOOK: query: -- disable sorted dynamic partition optimization to make sure the results are correct
+drop table t1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t1
+POSTHOOK: Output: default@t1
+PREHOOK: query: create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1(
+a string,
+b int,
+c int,
+d string)
+partitioned by (e string)
+clustered by(a)
+sorted by(a desc) into 10 buckets stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1_staging
+PREHOOK: Input: default@t1_staging@e=epart
+PREHOOK: Output: default@t1
+POSTHOOK: query: insert overwrite table t1 partition(e) select a,b,c,d,'epart' from t1_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1_staging
+POSTHOOK: Input: default@t1_staging@e=epart
+POSTHOOK: Output: default@t1@e=epart
+POSTHOOK: Lineage: t1 PARTITION(e=epart).a SIMPLE [(t1_staging)t1_staging.FieldSchema(name:a, type:string, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).b SIMPLE [(t1_staging)t1_staging.FieldSchema(name:b, type:int, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).c SIMPLE [(t1_staging)t1_staging.FieldSchema(name:c, type:int, comment:null), ]
+POSTHOOK: Lineage: t1 PARTITION(e=epart).d SIMPLE [(t1_staging)t1_staging.FieldSchema(name:d, type:string, comment:null), ]
+PREHOOK: query: select 'bucket_0'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_0
+fffdc9482d8a88626887d92851951b92e58dba2cfa7e22caeafe98baac4d2a3e042015-01-21
+fffd4906f05ab50dfa41f4ab226e83de981fd8aa4ce0d4f516e7ebf9566a4213\N\N2015-01-21
+fffd37a9ebbef0ed3d3440f7bacd64b823df685c90c827a75a5e9556126a8d0c\N\N2015-01-21
+fffce560fc7c69c5185eda181759a62f30f908c55f04291d612039a3ca16357a\N\N2015-01-21
+fffc502293653c4249cdeb8fad4ba2437006b56d527eae844d18e6d6324e424d002015-01-21
+fffad5028e769fdf811eea954785629c064826a5a008253136dbd4f189334827\N\N2015-01-21
+fff3474e56ee23c0df629b538268a438d74da36208bdb114bda2da4253f0b4c9\N\N2015-01-21
+fff2de7c5a9929257705996690aa45e5848e57a85eb91f83dfcfa6ac4219ae1b\N\N2015-01-21
+PREHOOK: query: select 'bucket_2'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_2
+fffcf74695669d1f444936243869f3586418c6d61d3abb5cc9acb67a3ad7bd2f002015-01-21
+fffbe3c110c390ec20218e5ad4a026ff515668ed55488b717319b556daa962a1002015-01-21
+fff876f50b99444aa7ef901383336299c47f035b87d253283f737d88a05edce8\N\N2015-01-21
+fff66e6646a820cf6d39ba467a735db4ffe4df344256bb1a1cb854086e766810\N\N2015-01-21
+fff4166378aa9d94cd4f8a9cd543375890a61b4f09a57dbfb31a66b33b3e3fd9\N\N2015-01-21
+fff06e59d471f90aa055c3089c1d8e380bbb86d5642142ecd0589b544dd44688002015-01-21
+fff03007f38c32085bb4c9389270b965d371168032845555b663b7d4653ec8ee\N\N2015-01-21
+PREHOOK: query: select 'bucket_4'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_4
+fffc0450ec9b28bae495dffc87a37cc3eea6d2d067ccea8b333185d28847ae2a\N\N2015-01-21
+fffad1074d813e6db5c23302a9170fe472c2968844499c90445cbc8559d64fe1082015-01-21
+fff56191e39b15f0e2f04984c70152fb1bde2ecba52ff5a73b4c28bf4d58c017002015-01-21
+fff201d3ead782d817f222c73cd241df1683cecdb109a5a22b02004b7b228b14\N\N2015-01-21
+fff0fbdf2bf7bfed9ea314bfa7fa14bf179345dde9ea79838f66acd70b943f0b\N\N2015-01-21
+PREHOOK: query: select 'bucket_6'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_6'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_6
+ffff67aef705abda0b89d899e408c28ef230fd0bb2cb0bb23b057e946ba9ca91\N\N2015-01-21
+fffe069c9bba031c86f1f49ec87bf75d4feb5cc5c7d430282f739ebc4337c673002015-01-21
+fffcb494aa56beb88fddb83cc3b0296d417d3ab7782be76c8c12d33e3f3d6a3c\N\N2015-01-21
+fffb1b226efc3cfaac8d73647ce4fa4e82413d67265fb55366ac3a4996518738012015-01-21
+fff4cabcf8a5f2f8edd27d5d92659ad80e3513d15ac2df7557a82ade5ed80b86\N\N2015-01-21
+fff42e4d4463fa981df990041ba914deaa1d1c1cdcb1547678ab899e34248526012015-01-21
+ffefa017a261a0b3e94c2386e0c47a015e2095e5d56b0f4fca2033d9755c9e45012015-01-21
+ffef34be95399a087af57dd713ca3510377625811bb4232473c1ba223239f284002015-01-21
+PREHOOK: query: select 'bucket_8'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'bucket_8'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+bucket_8
+fffee943d640a7714d09f9bd50dba08a9d0ebdd146655e4642c293a4396cb385\N\N2015-01-21
+fffc8a89a8406ba3ed651fc9248fe9b2afb1ee672999ef75d2d9c42484435bca\N\N2015-01-21
+fff06c6e0fd675ebeff09350e6b7a3900115f72341fd353e5e185e8983d10534002015-01-21
+ffef4b7433ee6008e389a1a4121a82b828123864e563a2afe67dcf29e2b71591\N\N2015-01-21
[17/61] [abbrv] hive git commit: HIVE-10965 : direct SQL for stats
fails in 0-column case (Sergey Shelukhin reviewed by Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-10965 : direct SQL for stats fails in 0-column case (Sergey Shelukhin reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fff8231
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fff8231
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fff8231
Branch: refs/heads/spark
Commit: 5fff8231ec88e3bff9db22541bcbf9dcc1c68361
Parents: 40f348f
Author: Thejas Nair <th...@hortonworks.com>
Authored: Wed Jun 10 15:10:57 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Wed Jun 10 15:10:57 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java | 1 +
.../java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 2 ++
ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java | 3 ++-
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index f5816a0..4891d42 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2060,6 +2060,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
@Override
public AggrStats getAggrColStatsFor(String dbName, String tblName,
List<String> colNames, List<String> partNames) throws NoSuchObjectException, MetaException, TException {
+ if (colNames.isEmpty()) return null; // Nothing to aggregate.
PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partNames);
return client.get_aggr_stats_for(req);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 0a56bac..ed810d2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1100,6 +1100,7 @@ class MetaStoreDirectSql {
public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation)
throws MetaException {
+ if (colNames.isEmpty() || partNames.isEmpty()) return new AggrStats(); // Nothing to aggregate.
long partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
List<ColumnStatisticsObj> colStatsList;
// Try to read from the cache first
@@ -1160,6 +1161,7 @@ class MetaStoreDirectSql {
private long partsFoundForPartitions(String dbName, String tableName,
List<String> partNames, List<String> colNames) throws MetaException {
+ assert !colNames.isEmpty() && !partNames.isEmpty();
long partsFound = 0;
boolean doTrace = LOG.isDebugEnabled();
String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""
http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 6615a28..0bd7f0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -250,7 +250,8 @@ public class StatsUtils {
neededColumns = processNeededColumns(schema, neededColumns);
AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
neededColumns, partNames);
- if (null == aggrStats) {
+ if (null == aggrStats || null == aggrStats.getColStats()
+ || aggrStats.getColStatsSize() == 0) {
// There are some partitions with no state (or we didn't fetch any state).
// Update the stats with empty list to reflect that in the
// state/initialize structures.
[57/61] [abbrv] hive git commit: HIVE-11050:
testCliDriver_vector_outer_join.* failures in Unit tests due to unstable data
creation queries (Matt McCline reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/vector_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/vector_outer_join1.q.out
index 2d3f2d8..8e8f7fa 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join1.q.out
@@ -1,43 +1,92 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +95,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +110,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +133,29 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select *
from small_alltypesorc_a c
@@ -114,11 +184,11 @@ STAGE PLANS:
$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col2 (type: int)
@@ -129,11 +199,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -141,10 +211,10 @@ STAGE PLANS:
0 _col2 (type: int)
1 _col2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3652 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3652 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -177,21 +247,25 @@ left outer join small_alltypesorc_a cd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
--51 NULL -1071480828 -1401575336 -51.0 NULL aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true -51 NULL -1071480828 -1401575336 -51.0 NULL aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL -64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL -64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL -64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL -64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL -64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--64 15601 NULL 1421812187 -64.0 15601.0 NULL 12AEw 1969-12-31 16:00:05.997 1969-12-31 15:59:58.456 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--64 15601 NULL 1960950366 -64.0 15601.0 NULL c0ODOF64 1969-12-31 16:00:13.274 1969-12-31 15:59:58.456 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-11 NULL -1069736047 -453772520 11.0 NULL k17Am8uPHWk02cEf1jet qrXLLNX1 1969-12-31 16:00:02.351 NULL true true 11 NULL -1069736047 -453772520 11.0 NULL k17Am8uPHWk02cEf1jet qrXLLNX1 1969-12-31 16:00:02.351 NULL true true
-11 NULL -1072910839 2048385991 11.0 NULL 0iqrc5 KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false 11 NULL -1072910839 2048385991 11.0 NULL 0iqrc5 KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false
-11 NULL -1073279343 -1595604468 11.0 NULL oj1YrV5Wa P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true 11 NULL -1073279343 -1595604468 11.0 NULL oj1YrV5Wa P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true
-8 NULL -1071363017 1349676361 8.0 NULL Anj0oF IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true 8 NULL -1071363017 1349676361 8.0 NULL Anj0oF IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true
-NULL -5470 -1072076362 1864027286 NULL -5470.0 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true NULL -5470 -1072076362 1864027286 NULL -5470.0 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true
-NULL -7382 -1073051226 -1887561756 NULL -7382.0 A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false NULL -7382 -1073051226 -1887561756 NULL -7382.0 A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false
-NULL -741 -1070883071 -1645852809 NULL -741.0 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false NULL -741 -1070883071 -1645852809 NULL -741.0 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false
-NULL -947 -1070551679 1864027286 NULL -947.0 iUR3Q 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:08.058 false true NULL -947 -1070551679 1864027286 NULL -947.0 iUR3Q 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:08.058 false true
-NULL 8373 -1072081801 1864027286 NULL 8373.0 dPkN74F7 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true NULL 8373 -1072081801 1864027286 NULL 8373.0 dPkN74F7 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL -64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL -64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL -64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL -64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select c.ctinyint
from small_alltypesorc_a c
@@ -220,11 +294,11 @@ STAGE PLANS:
$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col0 (type: tinyint)
@@ -235,11 +309,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -247,10 +321,10 @@ STAGE PLANS:
0 _col0 (type: tinyint)
1 _col0 (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3652 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3652 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -283,7 +357,6 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
--51
-64
-64
-64
@@ -309,16 +382,81 @@ POSTHOOK: Input: default@small_alltypesorc_a
-64
-64
-64
-11
-11
-11
-11
-11
-11
-11
-11
-11
-8
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
NULL
NULL
NULL
@@ -361,11 +499,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col1 (type: int)
@@ -373,11 +511,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_2:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col0 (type: tinyint)
@@ -388,11 +526,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3320 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -400,7 +538,7 @@ STAGE PLANS:
0 _col1 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3652 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -408,7 +546,7 @@ STAGE PLANS:
0 _col0 (type: tinyint)
1 _col0 (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4017 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
mode: hash
@@ -465,4 +603,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-41 -1544
+145 -8960
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/vector_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/vector_outer_join2.q.out
index 096ce7b..c649883 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join2.q.out
@@ -1,43 +1,97 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and cbigint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and cbigint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+NULL -13166 626923679 NULL NULL -13166.0 821UdmGbkEf4j NULL 1969-12-31 15:59:55.089 1969-12-31 16:00:15.69 true NULL
+NULL -14426 626923679 NULL NULL -14426.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.505 1969-12-31 16:00:13.309 true NULL
+NULL -14847 626923679 NULL NULL -14847.0 821UdmGbkEf4j NULL 1969-12-31 16:00:00.612 1969-12-31 15:59:43.704 true NULL
+NULL -15632 528534767 NULL NULL -15632.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:53.593 true NULL
+NULL -15830 253665376 NULL NULL -15830.0 1cGVWH7n1QU NULL 1969-12-31 16:00:02.582 1969-12-31 16:00:00.518 true NULL
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+-60 -200 NULL NULL -60.0 -200.0 NULL NULL 1969-12-31 16:00:11.996 1969-12-31 15:59:55.451 NULL NULL
+-61 -7196 NULL NULL -61.0 -7196.0 NULL 8Mlns2Tl6E0g 1969-12-31 15:59:44.823 1969-12-31 15:59:58.174 NULL false
+-61 -7196 NULL NULL -61.0 -7196.0 NULL fUJIN 1969-12-31 16:00:11.842 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL jf1Cw6qhkNToQuud 1969-12-31 16:00:12.388 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL yLiOchx5PfDTFdcMduBTg 1969-12-31 16:00:02.373 1969-12-31 15:59:58.174 NULL false
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +100,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +115,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +138,34 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-60 -200 NULL NULL -60.0 -200.0 NULL NULL 1969-12-31 16:00:11.996 1969-12-31 15:59:55.451 NULL NULL
+-61 -7196 NULL NULL -61.0 -7196.0 NULL 8Mlns2Tl6E0g 1969-12-31 15:59:44.823 1969-12-31 15:59:58.174 NULL false
+-61 -7196 NULL NULL -61.0 -7196.0 NULL fUJIN 1969-12-31 16:00:11.842 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL jf1Cw6qhkNToQuud 1969-12-31 16:00:12.388 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL yLiOchx5PfDTFdcMduBTg 1969-12-31 16:00:02.373 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+NULL -13166 626923679 NULL NULL -13166.0 821UdmGbkEf4j NULL 1969-12-31 15:59:55.089 1969-12-31 16:00:15.69 true NULL
+NULL -14426 626923679 NULL NULL -14426.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.505 1969-12-31 16:00:13.309 true NULL
+NULL -14847 626923679 NULL NULL -14847.0 821UdmGbkEf4j NULL 1969-12-31 16:00:00.612 1969-12-31 15:59:43.704 true NULL
+NULL -15632 528534767 NULL NULL -15632.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:53.593 true NULL
+NULL -15830 253665376 NULL NULL -15830.0 1cGVWH7n1QU NULL 1969-12-31 16:00:02.582 1969-12-31 16:00:00.518 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select count(*), sum(t1.c_cbigint) from (select c.cbigint as c_cbigint
from small_alltypesorc_a c
@@ -123,11 +203,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col0 (type: int)
@@ -135,11 +215,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_2:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col1 (type: bigint)
@@ -150,11 +230,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4182 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -162,7 +242,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 4600 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -170,11 +250,11 @@ STAGE PLANS:
0 _col1 (type: bigint)
1 _col0 (type: bigint)
outputColumnNames: _col1
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
mode: hash
@@ -231,4 +311,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21 17110062350
+34 -26289186744
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/vector_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
index 6eca0b5..502f2d3 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
@@ -1,43 +1,97 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and cstring1 in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and cstring1 in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-51 NULL NULL -1731061911 -51.0 NULL Pw53BBJ yL443x2437PO5Hv1U3lCjq2D 1969-12-31 16:00:08.451 NULL true false
+-51 NULL NULL -1846191223 -51.0 NULL Ul085f84S33Xd32u x1JC58g0Ukp 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1874052220 -51.0 NULL c61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULL false false
+-51 NULL NULL -1927203921 -51.0 NULL 45ja5suO 42S0I0 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1970551565 -51.0 NULL r2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULL false false
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+-51 NULL -31312632 1086455747 -51.0 NULL NULL Bc7xt12568c451o64LF5 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -337975743 608681041 -51.0 NULL NULL Ih2r28o6 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -413196097 -306198070 -51.0 NULL NULL F53QcSDPpxYF1Ub 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL -591488718 803603078 -51.0 NULL NULL X616UtmmA3FHan 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -738306196 -460430946 -51.0 NULL NULL dBOqv 1969-12-31 16:00:08.451 NULL NULL false
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +100,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +115,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +138,34 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-51 NULL -31312632 1086455747 -51.0 NULL NULL Bc7xt12568c451o64LF5 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -337975743 608681041 -51.0 NULL NULL Ih2r28o6 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -413196097 -306198070 -51.0 NULL NULL F53QcSDPpxYF1Ub 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL -591488718 803603078 -51.0 NULL NULL X616UtmmA3FHan 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -738306196 -460430946 -51.0 NULL NULL dBOqv 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL NULL -1731061911 -51.0 NULL Pw53BBJ yL443x2437PO5Hv1U3lCjq2D 1969-12-31 16:00:08.451 NULL true false
+-51 NULL NULL -1846191223 -51.0 NULL Ul085f84S33Xd32u x1JC58g0Ukp 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1874052220 -51.0 NULL c61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULL false false
+-51 NULL NULL -1927203921 -51.0 NULL 45ja5suO 42S0I0 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1970551565 -51.0 NULL r2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULL false false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -123,11 +203,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col0 (type: int)
@@ -135,11 +215,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_2:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col1 (type: string)
@@ -150,11 +230,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -162,14 +242,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 4840 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
keys:
0 _col1 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5324 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -226,7 +306,7 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-15
+20
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -264,11 +344,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col1 (type: string)
@@ -276,11 +356,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_2:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col0 (type: string)
@@ -291,11 +371,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -303,14 +383,14 @@ STAGE PLANS:
0 _col1 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 4840 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5324 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -367,7 +447,7 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21
+28
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -405,11 +485,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col3 (type: string), _col1 (type: bigint)
@@ -417,11 +497,11 @@ STAGE PLANS:
$hdt$_0:$hdt$_2:c
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col2 (type: string), _col0 (type: int)
@@ -432,11 +512,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -444,14 +524,14 @@ STAGE PLANS:
0 _col3 (type: string), _col1 (type: bigint)
1 _col1 (type: string), _col0 (type: bigint)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 4840 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
keys:
0 _col2 (type: string), _col0 (type: int)
1 _col1 (type: string), _col0 (type: int)
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5324 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -508,4 +588,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21
+28
[38/61] [abbrv] hive git commit: HIVE-10940 :
HiveInputFormat::pushFilters serializes PPD objects for each getRecordReader
call (Sergey Shelukhin, reviewed by Prasanth J, Gopal V)
Posted by xu...@apache.org.
HIVE-10940 : HiveInputFormat::pushFilters serializes PPD objects for each getRecordReader call (Sergey Shelukhin, reviewed by Prasanth J, Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b080e061
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b080e061
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b080e061
Branch: refs/heads/spark
Commit: b080e061d48e3058be9a3f01083b46d69c65e882
Parents: ed7b70a
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jun 15 17:18:19 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jun 15 17:18:19 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/HiveInputFormat.java | 38 +++++++++++---------
.../hadoop/hive/ql/plan/TableScanDesc.java | 20 ++++++++++-
2 files changed, 41 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b080e061/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 54e1d4e..2ff3951 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -450,25 +450,31 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return;
}
- Serializable filterObject = scanDesc.getFilterObject();
- if (filterObject != null) {
- jobConf.set(
- TableScanDesc.FILTER_OBJECT_CONF_STR,
- Utilities.serializeObject(filterObject));
+ String serializedFilterObj = scanDesc.getSerializedFilterObject();
+ String serializedFilterExpr = scanDesc.getSerializedFilterExpr();
+ boolean hasObj = serializedFilterObj != null, hasExpr = serializedFilterExpr != null;
+ if (!hasObj) {
+ Serializable filterObject = scanDesc.getFilterObject();
+ if (filterObject != null) {
+ serializedFilterObj = Utilities.serializeObject(filterObject);
+ }
+ }
+ if (serializedFilterObj != null) {
+ jobConf.set(TableScanDesc.FILTER_OBJECT_CONF_STR, serializedFilterObj);
+ }
+ if (!hasExpr) {
+ serializedFilterExpr = Utilities.serializeExpression(filterExpr);
}
-
String filterText = filterExpr.getExprString();
- String filterExprSerialized = Utilities.serializeExpression(filterExpr);
if (LOG.isDebugEnabled()) {
- LOG.debug("Filter text = " + filterText);
- LOG.debug("Filter expression = " + filterExprSerialized);
- }
- jobConf.set(
- TableScanDesc.FILTER_TEXT_CONF_STR,
- filterText);
- jobConf.set(
- TableScanDesc.FILTER_EXPR_CONF_STR,
- filterExprSerialized);
+ LOG.debug("Pushdown initiated with filterText = " + filterText + ", filterExpr = "
+ + filterExpr + ", serializedFilterExpr = " + serializedFilterExpr + " ("
+ + (hasExpr ? "desc" : "new") + ")" + (serializedFilterObj == null ? "" :
+ (", serializedFilterObj = " + serializedFilterObj + " (" + (hasObj ? "desc" : "new")
+ + ")")));
+ }
+ jobConf.set(TableScanDesc.FILTER_TEXT_CONF_STR, filterText);
+ jobConf.set(TableScanDesc.FILTER_EXPR_CONF_STR, serializedFilterExpr);
}
protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass,
http://git-wip-us.apache.org/repos/asf/hive/blob/b080e061/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 6282380..9e9a2a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.parse.TableSample;
@@ -72,6 +73,9 @@ public class TableScanDesc extends AbstractOperatorDesc {
private ExprNodeGenericFuncDesc filterExpr;
private transient Serializable filterObject;
+ private String serializedFilterExpr;
+ private String serializedFilterObject;
+
// Both neededColumnIDs and neededColumns should never be null.
// When neededColumnIDs is an empty list,
@@ -101,7 +105,6 @@ public class TableScanDesc extends AbstractOperatorDesc {
private transient final Table tableMetadata;
-
public TableScanDesc() {
this(null, null);
}
@@ -144,7 +147,11 @@ public class TableScanDesc extends AbstractOperatorDesc {
}
public void setFilterExpr(ExprNodeGenericFuncDesc filterExpr) {
+ // TODO: we could avoid serialization if it's the same expr. Check?
this.filterExpr = filterExpr;
+ if (filterExpr != null) {
+ serializedFilterExpr = Utilities.serializeExpression(filterExpr);
+ }
}
public Serializable getFilterObject() {
@@ -153,6 +160,9 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setFilterObject(Serializable filterObject) {
this.filterObject = filterObject;
+ if (filterObject != null) {
+ serializedFilterObject = Utilities.serializeObject(filterObject);
+ }
}
public void setNeededColumnIDs(List<Integer> neededColumnIDs) {
@@ -281,4 +291,12 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setTableSample(TableSample tableSample) {
this.tableSample = tableSample;
}
+
+ public String getSerializedFilterExpr() {
+ return serializedFilterExpr;
+ }
+
+ public String getSerializedFilterObject() {
+ return serializedFilterObject;
+ }
}
[20/61] [abbrv] hive git commit: HIVE-10956: HS2 leaks HMS
connections (Jimmy, reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10956: HS2 leaks HMS connections (Jimmy, reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ac49574f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ac49574f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ac49574f
Branch: refs/heads/spark
Commit: ac49574f278d3b23b44c9fde74f89cc885d780cb
Parents: 9f263fc
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Thu Jun 4 14:52:40 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Thu Jun 11 08:45:58 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/HiveMetaStoreClient.java | 12 +++++----
.../apache/hadoop/hive/ql/metadata/Hive.java | 8 +++++-
.../service/cli/operation/SQLOperation.java | 16 +----------
.../hive/service/cli/session/HiveSession.java | 12 ++++++++-
.../service/cli/session/HiveSessionImpl.java | 28 +++++++++++++++++---
.../cli/session/HiveSessionImplwithUGI.java | 21 ---------------
.../service/cli/thrift/ThriftCLIService.java | 4 +++
7 files changed, 54 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 4891d42..a5f5053 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -40,6 +40,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.security.auth.login.LoginException;
@@ -170,6 +171,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
private Map<String, String> currentMetaVars;
+ private static final AtomicInteger connCount = new AtomicInteger(0);
+
// for thrift connects
private int retries = 5;
private long retryDelaySeconds = 0;
@@ -419,6 +422,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
client = new ThriftHiveMetastore.Client(protocol);
try {
transport.open();
+ LOG.info("Opened a connection to metastore, current connections: " + connCount.incrementAndGet());
isConnected = true;
} catch (TTransportException e) {
tte = e;
@@ -499,6 +503,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
// just in case, we make this call.
if ((transport != null) && transport.isOpen()) {
transport.close();
+ LOG.info("Closed a connection to metastore, current connections: " + connCount.decrementAndGet());
}
}
@@ -1974,19 +1979,16 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
private static class SynchronizedHandler implements InvocationHandler {
private final IMetaStoreClient client;
- private static final Object lock = SynchronizedHandler.class;
SynchronizedHandler(IMetaStoreClient client) {
this.client = client;
}
@Override
- public Object invoke(Object proxy, Method method, Object [] args)
+ public synchronized Object invoke(Object proxy, Method method, Object [] args)
throws Throwable {
try {
- synchronized (lock) {
- return method.invoke(client, args);
- }
+ return method.invoke(client, args);
} catch (InvocationTargetException e) {
throw e.getTargetException();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 8c948a9..0e990f6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.PartitionDropOptions;
@@ -3009,7 +3010,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
*/
@LimitedPrivate(value = {"Hive"})
@Unstable
- public IMetaStoreClient getMSC() throws MetaException {
+ public synchronized IMetaStoreClient getMSC() throws MetaException {
if (metaStoreClient == null) {
try {
owner = UserGroupInformation.getCurrentUser();
@@ -3019,6 +3020,11 @@ private void constructOneLBLocationMap(FileStatus fSta,
throw new MetaException(msg + "\n" + StringUtils.stringifyException(e));
}
metaStoreClient = createMetaStoreClient();
+ String metaStoreUris = conf.getVar(HiveConf.ConfVars.METASTOREURIS);
+ if (!org.apache.commons.lang3.StringUtils.isEmpty(metaStoreUris)) {
+ // get a synchronized wrapper if the meta store is remote.
+ metaStoreClient = HiveMetaStoreClient.newSynchronizedClient(metaStoreClient);
+ }
}
return metaStoreClient;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index 33ee16b..cc9df76 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.OperationLog;
@@ -186,7 +185,7 @@ public class SQLOperation extends ExecuteStatementOperation {
final SessionState parentSessionState = SessionState.get();
// ThreadLocal Hive object needs to be set in background thread.
// The metastore client in Hive is associated with right user.
- final Hive parentHive = getSessionHive();
+ final Hive parentHive = parentSession.getSessionHive();
// Current UGI will get used by metastore when metsatore is in embedded mode
// So this needs to get passed to the new background thread
final UserGroupInformation currentUGI = getCurrentUGI(opConfig);
@@ -261,19 +260,6 @@ public class SQLOperation extends ExecuteStatementOperation {
}
}
- /**
- * Returns the ThreadLocal Hive for the current thread
- * @return Hive
- * @throws HiveSQLException
- */
- private Hive getSessionHive() throws HiveSQLException {
- try {
- return Hive.get();
- } catch (HiveException e) {
- throw new HiveSQLException("Failed to get ThreadLocal Hive object", e);
- }
- }
-
private void registerCurrentOperationLog() {
if (isOperationLogEnabled) {
if (operationLog == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/service/src/java/org/apache/hive/service/cli/session/HiveSession.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSession.java b/service/src/java/org/apache/hive/service/cli/session/HiveSession.java
index 65f9b29..4f4e92d 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSession.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSession.java
@@ -22,8 +22,16 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.cli.*;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.FetchType;
+import org.apache.hive.service.cli.GetInfoType;
+import org.apache.hive.service.cli.GetInfoValue;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.TableSchema;
public interface HiveSession extends HiveSessionBase {
@@ -31,6 +39,8 @@ public interface HiveSession extends HiveSessionBase {
IMetaStoreClient getMetaStoreClient() throws HiveSQLException;
+ Hive getSessionHive() throws HiveSQLException;
+
/**
* getInfo operation handler
* @param getInfoType
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 343c68e..a2fae69 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -90,6 +90,9 @@ public class HiveSessionImpl implements HiveSession {
private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
private boolean isOperationLogEnabled;
private File sessionLogDir;
+
+ private Hive sessionHive;
+
private volatile long lastAccessTime;
private volatile long lastIdleTime;
@@ -142,6 +145,11 @@ public class HiveSessionImpl implements HiveSession {
LOG.error(msg, e);
throw new HiveSQLException(msg, e);
}
+ try {
+ sessionHive = Hive.get(getHiveConf());
+ } catch (HiveException e) {
+ throw new HiveSQLException("Failed to get metastore connection", e);
+ }
// Process global init file: .hiverc
processGlobalInitFile();
if (sessionConfMap != null) {
@@ -278,6 +286,7 @@ public class HiveSessionImpl implements HiveSession {
if (userAccess) {
lastAccessTime = System.currentTimeMillis();
}
+ Hive.set(sessionHive);
}
/**
@@ -326,13 +335,16 @@ public class HiveSessionImpl implements HiveSession {
}
@Override
+ public Hive getSessionHive() {
+ return sessionHive;
+ }
+
+ @Override
public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
try {
- return Hive.get(getHiveConf()).getMSC();
- } catch (HiveException e) {
- throw new HiveSQLException("Failed to get metastore connection", e);
+ return getSessionHive().getMSC();
} catch (MetaException e) {
- throw new HiveSQLException("Failed to get metastore connection", e);
+ throw new HiveSQLException("Error acquiring metastore connection", e);
}
}
@@ -579,6 +591,14 @@ public class HiveSessionImpl implements HiveSession {
}
sessionState = null;
}
+ if (sessionHive != null) {
+ try {
+ Hive.closeCurrent();
+ } catch (Throwable t) {
+ LOG.warn("Error closing sessionHive", t);
+ }
+ sessionHive = null;
+ }
release(true);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
index a29e5d1..56af643 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.service.auth.HiveAuthFactory;
@@ -43,7 +42,6 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl {
private UserGroupInformation sessionUgi = null;
private String delegationTokenStr = null;
- private Hive sessionHive = null;
private HiveSession proxySession = null;
static final Log LOG = LogFactory.getLog(HiveSessionImplwithUGI.class);
@@ -52,14 +50,6 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl {
super(protocol, username, password, hiveConf, ipAddress);
setSessionUGI(username);
setDelegationToken(delegationToken);
-
- // create a new metastore connection for this particular user session
- Hive.set(null);
- try {
- sessionHive = Hive.get(getHiveConf());
- } catch (HiveException e) {
- throw new HiveSQLException("Failed to setup metastore connection", e);
- }
}
// setup appropriate UGI for the session
@@ -87,15 +77,6 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl {
return this.delegationTokenStr;
}
- @Override
- protected synchronized void acquire(boolean userAccess) {
- super.acquire(userAccess);
- // if we have a metastore connection with impersonation, then set it first
- if (sessionHive != null) {
- Hive.set(sessionHive);
- }
- }
-
/**
* Close the file systems for the session and remove it from the FileSystem cache.
* Cancel the session's delegation token and close the metastore connection
@@ -146,8 +127,6 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl {
} catch (HiveException e) {
throw new HiveSQLException("Couldn't cancel delegation token", e);
}
- // close the metastore connection created with this delegation token
- Hive.closeCurrent();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ac49574f/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 5a0f1c8..dfb7faa 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -24,6 +24,7 @@ import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.security.auth.login.LoginException;
@@ -67,6 +68,7 @@ public abstract class ThriftCLIService extends AbstractService implements TCLISe
protected CLIService cliService;
private static final TStatus OK_STATUS = new TStatus(TStatusCode.SUCCESS_STATUS);
protected static HiveAuthFactory hiveAuthFactory;
+ private static final AtomicInteger sessionCount = new AtomicInteger();
protected int portNum;
protected InetAddress serverIPAddress;
@@ -304,6 +306,7 @@ public abstract class ThriftCLIService extends AbstractService implements TCLISe
if (context != null) {
context.setSessionHandle(sessionHandle);
}
+ LOG.info("Opened a session, current sessions: " + sessionCount.incrementAndGet());
} catch (Exception e) {
LOG.warn("Error opening session: ", e);
resp.setStatus(HiveSQLException.toTStatus(e));
@@ -446,6 +449,7 @@ public abstract class ThriftCLIService extends AbstractService implements TCLISe
try {
SessionHandle sessionHandle = new SessionHandle(req.getSessionHandle());
cliService.closeSession(sessionHandle);
+ LOG.info("Closed a session, current sessions: " + sessionCount.decrementAndGet());
resp.setStatus(OK_STATUS);
ThriftCLIServerContext context =
(ThriftCLIServerContext)currentServerContext.get();
[21/61] [abbrv] hive git commit: HIVE-10976 Redundant HiveMetaStore
connect check in HS2 CLIService start (Chaoyu via Jimmy)
Posted by xu...@apache.org.
HIVE-10976 Redundant HiveMetaStore connect check in HS2 CLIService start (Chaoyu via Jimmy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/671e1a71
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/671e1a71
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/671e1a71
Branch: refs/heads/spark
Commit: 671e1a71759108cd11099d9404942bf912f27b1b
Parents: ac49574
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Thu Jun 11 10:02:02 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Thu Jun 11 10:10:08 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/service/cli/CLIService.java | 13 -------------
1 file changed, 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/671e1a71/service/src/java/org/apache/hive/service/cli/CLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java
index a3af7b2..4c7d7f4 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIService.java
@@ -145,19 +145,6 @@ public class CLIService extends CompositeService implements ICLIService {
@Override
public synchronized void start() {
super.start();
- // Initialize and test a connection to the metastore
- IMetaStoreClient metastoreClient = null;
- try {
- metastoreClient = new HiveMetaStoreClient(hiveConf);
- metastoreClient.getDatabases("default");
- } catch (Exception e) {
- throw new ServiceException("Unable to connect to MetaStore!", e);
- }
- finally {
- if (metastoreClient != null) {
- metastoreClient.close();
- }
- }
}
@Override
[41/61] [abbrv] hive git commit: HIVE-11026 : Make vector_outer_join*
test more robust (Ashutosh Chauhan via Hari Sankar)
Posted by xu...@apache.org.
HIVE-11026 : Make vector_outer_join* test more robust (Ashutosh Chauhan via Hari Sankar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3792b7b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3792b7b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3792b7b
Branch: refs/heads/spark
Commit: a3792b7b785f0b8db3d557db31cd1f9b2cd0acdc
Parents: b86f10d
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Wed Jun 17 00:42:21 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jun 17 00:42:21 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/vector_outer_join1.q | 5 +-
.../queries/clientpositive/vector_outer_join2.q | 5 +-
.../queries/clientpositive/vector_outer_join3.q | 6 +-
.../clientpositive/tez/vector_outer_join1.q.out | 50 ++++++++--------
.../clientpositive/tez/vector_outer_join2.q.out | 24 ++++----
.../clientpositive/tez/vector_outer_join3.q.out | 62 ++++++++++----------
.../clientpositive/vector_outer_join1.q.out | 2 +
.../clientpositive/vector_outer_join2.q.out | 2 +
.../clientpositive/vector_outer_join3.q.out | 2 +
9 files changed, 87 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/queries/clientpositive/vector_outer_join1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join1.q b/ql/src/test/queries/clientpositive/vector_outer_join1.q
index 5a1aa0b..b695b08 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join1.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join1.q
@@ -13,7 +13,8 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q;
+ select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
@@ -61,4 +62,4 @@ left outer join small_alltypesorc_a cd
on cd.cint = c.cint
left outer join small_alltypesorc_a hd
on hd.ctinyint = c.ctinyint
-) t1;
\ No newline at end of file
+) t1;
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/queries/clientpositive/vector_outer_join2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join2.q b/ql/src/test/queries/clientpositive/vector_outer_join2.q
index b3503fd..d7abb21 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join2.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join2.q
@@ -14,7 +14,8 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q;
+ select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
@@ -36,4 +37,4 @@ left outer join small_alltypesorc_a cd
on cd.cint = c.cint
left outer join small_alltypesorc_a hd
on hd.cbigint = c.cbigint
-) t1;
\ No newline at end of file
+) t1;
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/queries/clientpositive/vector_outer_join3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join3.q b/ql/src/test/queries/clientpositive/vector_outer_join3.q
index a612ab0..f256e39 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join3.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join3.q
@@ -14,7 +14,9 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q;
+ select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1;
+
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
@@ -77,4 +79,4 @@ left outer join small_alltypesorc_a cd
on cd.cstring2 = c.cstring2 and cd.cbigint = c.cbigint
left outer join small_alltypesorc_a hd
on hd.cstring1 = c.cstring1 and hd.cint = c.cint
-) t1;
\ No newline at end of file
+) t1;
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
index 4101c35..95ed151 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -111,11 +113,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -125,11 +127,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
input vertices:
1 Map 2
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -139,16 +141,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
Execution mode: vectorized
@@ -218,11 +220,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -232,11 +234,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 2
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -246,16 +248,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Stage: Stage-0
@@ -357,11 +359,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -371,7 +373,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -382,7 +384,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count(), sum(_col0)
@@ -398,31 +400,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
index b8e18e0..31ef469 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -118,11 +120,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -132,7 +134,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -143,12 +145,12 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col1 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
mode: hash
@@ -163,31 +165,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
index b5619a0..75c4215 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -118,11 +120,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -132,7 +134,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -142,7 +144,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -158,31 +160,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -265,11 +267,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -279,7 +281,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -289,7 +291,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -305,31 +307,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -412,11 +414,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -426,7 +428,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col2
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -436,7 +438,7 @@ STAGE PLANS:
1 _col1 (type: string), _col0 (type: int)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4578 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -452,31 +454,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: bigint)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/vector_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/vector_outer_join1.q.out
index c7c89ba..2d3f2d8 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join1.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/vector_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/vector_outer_join2.q.out
index ff2e52a..096ce7b 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join2.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
http://git-wip-us.apache.org/repos/asf/hive/blob/a3792b7b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
index 901808c..6eca0b5 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
@@ -46,6 +46,7 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -61,6 +62,7 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
+order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
[32/61] [abbrv] hive git commit: HIVE-10685: Alter table concatenate
oparetor will cause duplicate data (guoliming reviewed by Prasanth
Jayachandran)
Posted by xu...@apache.org.
HIVE-10685: Alter table concatenate oparetor will cause duplicate data (guoliming reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3934c188
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3934c188
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3934c188
Branch: refs/heads/spark
Commit: 3934c18861072299e83d0749cf64c6411399bdeb
Parents: 4f28731
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Jun 15 15:25:24 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Jun 15 15:25:24 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3934c188/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
index 603b75a..cf6fa2a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
@@ -92,6 +92,7 @@ public class OrcFileStripeMergeRecordReader implements
keyWrapper.setRowIndexStride(reader.getRowIndexStride());
keyWrapper.setTypes(reader.getTypes());
} else {
+ stripeIdx++;
continue;
}
return true;
[05/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out b/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
index 2fb148a..02e24c3 100644
--- a/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
@@ -97,11 +97,11 @@ STAGE PLANS:
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string)
- outputColumnNames: l_shipdate
+ outputColumnNames: _col0
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(l_shipdate)
- keys: l_shipdate (type: string)
+ aggregations: count(_col0)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
@@ -264,15 +264,15 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
- outputColumnNames: l_shipdate, _count_of_l_shipdate
+ outputColumnNames: _col0, _count_of_l_shipdate
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_count_of_l_shipdate)
- keys: l_shipdate (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
@@ -447,12 +447,12 @@ STAGE PLANS:
alias: lineitem_ix
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: l_shipdate (type: string)
- outputColumnNames: l_shipdate
+ expressions: year(l_shipdate) (type: int), month(l_shipdate) (type: int), l_shipdate (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(l_shipdate)
- keys: year(l_shipdate) (type: int), month(l_shipdate) (type: int)
+ aggregations: count(_col2)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 120 Data size: 12099 Basic stats: COMPLETE Column stats: NONE
@@ -585,15 +585,15 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
- outputColumnNames: l_shipdate, _count_of_l_shipdate
+ expressions: year(l_shipdate) (type: int), month(l_shipdate) (type: int), l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _count_of_l_shipdate
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_count_of_l_shipdate)
- keys: year(l_shipdate) (type: int), month(l_shipdate) (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
@@ -912,15 +912,15 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: null-subquery1:dummy-subquery1:default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: null-subquery1:$hdt$_0-subquery1:$hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
- outputColumnNames: l_shipdate, _count_of_l_shipdate
+ outputColumnNames: _col0, _count_of_l_shipdate
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_count_of_l_shipdate)
- keys: l_shipdate (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
@@ -1004,11 +1004,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 1 (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: count(key)
- keys: key (type: int)
+ aggregations: count(_col0)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1058,15 +1058,15 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
- outputColumnNames: key, _count_of_key
+ outputColumnNames: _col0, _count_of_key
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: sum(_count_of_key)
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1168,11 +1168,11 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
- outputColumnNames: key, _count_of_key
+ outputColumnNames: _col0, _count_of_key
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: sum(_count_of_key)
@@ -1226,10 +1226,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1281,10 +1281,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: value (type: int), key (type: int)
- outputColumnNames: value, key
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: value (type: int), key (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1343,10 +1343,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 3 (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1401,10 +1401,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1455,11 +1455,11 @@ STAGE PLANS:
alias: tbl
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: key (type: int)
- outputColumnNames: key
+ expressions: key (type: int), substr(key, 2, 3) (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), substr(key, 2, 3) (type: string)
+ keys: _col0 (type: int), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1515,10 +1515,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: value (type: int), key (type: int)
- outputColumnNames: value, key
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: value (type: int), key (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1577,10 +1577,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), 1 (type: int)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), value (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1632,10 +1632,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1687,10 +1687,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1742,10 +1742,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1797,10 +1797,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), value (type: int)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), value (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1855,10 +1855,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), 2 (type: int)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), value (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1913,10 +1913,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: 3 (type: int), 2 (type: int)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), value (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -1969,16 +1969,20 @@ STAGE PLANS:
Filter Operator
predicate: (value = key) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Group By Operator
- keys: key (type: int), value (type: int)
- mode: hash
+ Select Operator
+ expressions: key (type: int), value (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -2023,16 +2027,20 @@ STAGE PLANS:
Filter Operator
predicate: (value = key) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Group By Operator
- keys: key (type: int), substr(value, 2, 3) (type: string)
- mode: hash
+ Select Operator
+ expressions: key (type: int), substr(value, 2, 3) (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -2075,11 +2083,11 @@ STAGE PLANS:
alias: tbl
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: key (type: int), value (type: int)
- outputColumnNames: key, value
+ expressions: key (type: int), substr(value, 2, 3) (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), substr(value, 2, 3) (type: string)
+ keys: _col0 (type: int), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -2134,10 +2142,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), 2 (type: int)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: key (type: int), value (type: int)
+ keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -2153,17 +2161,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), 2 (type: int)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -2279,18 +2283,22 @@ STAGE PLANS:
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(key)
- keys: key (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Group By Operator
+ aggregations: count(_col0)
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -2373,18 +2381,18 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__tblpart_tbl_part_index__
+ alias: $hdt$_0:default__tblpart_tbl_part_index__
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
- outputColumnNames: key, _count_of_key
+ outputColumnNames: _col0, _count_of_key
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_count_of_key)
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -2488,11 +2496,11 @@ STAGE PLANS:
Statistics: Num rows: 17 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 17 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(key)
- keys: key (type: int)
+ aggregations: count(_col0)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 17 Data size: 70 Basic stats: COMPLETE Column stats: NONE
@@ -2568,15 +2576,15 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
- outputColumnNames: key, _count_of_key
+ outputColumnNames: _col0, _count_of_key
Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_count_of_key)
- keys: key (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out
index 36af99a..12f1abb 100644
--- a/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/router_join_ppr.q.out
@@ -118,29 +118,36 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ alias: b
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string), _col2 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -148,11 +155,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -162,55 +171,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
+ name: default.srcpart
+ name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-08
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -251,12 +234,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
- ds 2008-04-08
- hr 12
+ ds 2008-04-09
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -297,12 +280,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-09
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -341,14 +324,41 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ /srcpart/ds=2008-04-09/hr=11 [b]
+ /srcpart/ds=2008-04-09/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -356,13 +366,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -372,48 +380,48 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Left Outer Join0 to 1
filter mappings:
- 1 [0, 1]
+ 0 [1, 1]
filter predicates:
- 0
- 1 {(VALUE._col1 = '2008-04-08')}
+ 0 {(VALUE._col1 = '2008-04-08')}
+ 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 244 Data size: 2593 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 10.0) and (UDFToDouble(_col3) < 20.0)) (type: boolean)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -607,16 +615,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -723,16 +735,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -789,16 +805,16 @@ STAGE PLANS:
condition map:
Right Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col7, _col8
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ predicate: ((UDFToDouble(_col0) > 10.0) and (UDFToDouble(_col0) < 20.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col7 (type: string), _col8 (type: string)
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -983,29 +999,36 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ alias: b
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1013,11 +1036,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -1027,55 +1052,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
+ name: default.srcpart
+ name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-08
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1114,14 +1113,39 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1129,13 +1153,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -1145,41 +1167,43 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 10.0) and (UDFToDouble(_col3) < 20.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -1365,20 +1389,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
+ predicate: (((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and (UDFToDouble(key) > 15.0)) and (UDFToDouble(key) < 25.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1474,103 +1502,9 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
-#### A masked pattern was here ####
- Partition
- base file name: hr=11
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 11
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- numFiles 1
- numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
- rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
-#### A masked pattern was here ####
- Partition
- base file name: hr=12
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- numFiles 1
- numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
- rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
Truncated Path -> Alias:
/srcpart/ds=2008-04-08/hr=11 [a]
/srcpart/ds=2008-04-08/hr=12 [a]
- /srcpart/ds=2008-04-09/hr=11 [a]
- /srcpart/ds=2008-04-09/hr=12 [a]
Map 3
Map Operator Tree:
TableScan
@@ -1579,16 +1513,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: (((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1643,41 +1581,37 @@ STAGE PLANS:
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col2, _col7, _col8
- Statistics: Num rows: 244 Data size: 2593 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: (((_col0 > 10) and (_col0 < 20)) and (_col2 = '2008-04-08')) (type: boolean)
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col7 (type: string), _col8 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1,_col2,_col3
- columns.types string:string:string:string
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string:string:string:string
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
Stage: Stage-0
Fetch Operator
@@ -1697,8 +1631,6 @@ PREHOOK: Input: default@src
PREHOOK: Input: default@srcpart
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: FROM
srcpart a
@@ -1712,8 +1644,6 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@srcpart
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
17 val_17 17 val_17
17 val_17 17 val_17
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out
index c9c09e8..1f6aac2 100644
--- a/ql/src/test/results/clientpositive/spark/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out
@@ -2516,14 +2516,18 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value is not null and (key > 100)) (type: boolean)
- Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 3 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: int)
+ predicate: ((key > 100) and value is not null) (type: boolean)
+ Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 4 Data size: 29 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -2534,10 +2538,10 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
- outputColumnNames: value
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: value (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
@@ -2552,7 +2556,7 @@ STAGE PLANS:
condition map:
Left Semi Join 0 to 1
keys:
- 0 value (type: string)
+ 0 _col1 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 49 Basic stats: COMPLETE Column stats: NONE
[02/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_8.q.out b/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
index 7aee2af..f580bd8 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
@@ -88,11 +88,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -108,7 +108,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(1) (type: bigint)
+ expressions: key (type: string), 1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -125,7 +125,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(2) (type: bigint)
+ expressions: key (type: string), 2 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
index 87ceca2..0931d11 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
@@ -95,11 +95,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_top_level.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_top_level.q.out b/ql/src/test/results/clientpositive/spark/union_top_level.q.out
index 5136989..dede1ef 100644
--- a/ql/src/test/results/clientpositive/spark/union_top_level.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_top_level.q.out
@@ -37,7 +37,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 0) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 0 (type: int)
@@ -56,7 +56,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 1) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 1 (type: int)
@@ -75,7 +75,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 2) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 2 (type: int)
@@ -414,7 +414,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 0) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 0 (type: int)
@@ -433,7 +433,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 1) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 1 (type: int)
@@ -452,7 +452,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 2) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 2 (type: int)
@@ -616,7 +616,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 0) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 0 (type: int)
@@ -635,7 +635,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 1) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 1 (type: int)
@@ -654,7 +654,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 2) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 2 (type: int)
@@ -805,7 +805,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 0) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 0 (type: int)
@@ -824,7 +824,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 1) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 1.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 1 (type: int)
@@ -843,7 +843,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key % 3) = 2) (type: boolean)
+ predicate: ((UDFToDouble(key) % 3.0) = 2.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), 2 (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
index aaac8aa..9e1402b 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
@@ -133,11 +133,11 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: i (type: int)
- outputColumnNames: i
+ outputColumnNames: _col0
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(50), avg(UDFToDouble(50)), avg(CAST( 50 AS decimal(10,0)))
- keys: i (type: int)
+ aggregations: avg(50), avg(50.0), avg(50)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.8.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.8.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.8.out
index 44ecd09..69f4754 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.8.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.8.out
@@ -133,11 +133,11 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: i (type: int)
- outputColumnNames: i
+ outputColumnNames: _col0
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(50), avg(UDFToDouble(50)), avg(CAST( 50 AS decimal(10,0)))
- keys: i (type: int)
+ aggregations: avg(50), avg(50.0), avg(50)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1049 Data size: 311170 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
index cb2d56b..7e161d1 100644
--- a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
@@ -54,11 +54,11 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14))
- outputColumnNames: cint, cdecimal1, cdecimal2
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), count()
- keys: cint (type: int)
+ aggregations: count(_col1), max(_col1), min(_col1), sum(_col1), count(_col2), max(_col2), min(_col2), sum(_col2), count()
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
@@ -159,11 +159,11 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdecimal1 (type: decimal(20,10)), cdecimal2 (type: decimal(23,14))
- outputColumnNames: cint, cdecimal1, cdecimal2
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdecimal1), max(cdecimal1), min(cdecimal1), sum(cdecimal1), avg(cdecimal1), stddev_pop(cdecimal1), stddev_samp(cdecimal1), count(cdecimal2), max(cdecimal2), min(cdecimal2), sum(cdecimal2), avg(cdecimal2), stddev_pop(cdecimal2), stddev_samp(cdecimal2), count()
- keys: cint (type: int)
+ aggregations: count(_col1), max(_col1), min(_col1), sum(_col1), avg(_col1), stddev_pop(_col1), stddev_samp(_col1), count(_col2), max(_col2), min(_col2), sum(_col2), avg(_col2), stddev_pop(_col2), stddev_samp(_col2), count()
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
Statistics: Num rows: 12288 Data size: 2165060 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
index 23f2b98..322270f 100644
--- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
@@ -129,10 +129,10 @@ STAGE PLANS:
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s (type: string), t (type: tinyint)
- outputColumnNames: s, t
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: s (type: string), t (type: tinyint)
+ keys: _col0 (type: string), _col1 (type: tinyint)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_elt.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_elt.q.out b/ql/src/test/results/clientpositive/spark/vector_elt.q.out
index 818e266..49d1458 100644
--- a/ql/src/test/results/clientpositive/spark/vector_elt.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_elt.q.out
@@ -21,7 +21,7 @@ STAGE PLANS:
predicate: (ctinyint > 0) (type: boolean)
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ((ctinyint % 2) + 1) (type: int), cstring1 (type: string), cint (type: int), elt(((ctinyint % 2) + 1), cstring1, cint) (type: string)
+ expressions: ((UDFToInteger(ctinyint) % 2) + 1) (type: int), cstring1 (type: string), cint (type: int), elt(((UDFToInteger(ctinyint) % 2) + 1), cstring1, cint) (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Limit
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
index 6f62820..3d6a236 100644
--- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
@@ -129,11 +129,11 @@ STAGE PLANS:
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s (type: string), t (type: tinyint), b (type: bigint)
- outputColumnNames: s, t, b
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(b)
- keys: s (type: string), t (type: tinyint)
+ aggregations: max(_col2)
+ keys: _col0 (type: string), _col1 (type: tinyint)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index 02c1fc6..8cf1a81 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -30,23 +30,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-2
Spark
+ Edges:
+ Reducer 4 <- Map 3 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 3
- Map Operator Tree:
- TableScan
- alias: li
- Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((l_partkey is not null and l_orderkey is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: int)
- 1 l_partkey (type: int)
- Local Work:
- Map Reduce Local Work
- Map 4
+ Map 2
Map Operator Tree:
TableScan
alias: lineitem
@@ -65,18 +53,11 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
-
- Stage: Stage-1
- Spark
- Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
-#### A masked pattern was here ####
- Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
alias: lineitem
@@ -84,17 +65,21 @@ STAGE PLANS:
Filter Operator
predicate: l_partkey is not null (type: boolean)
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: l_partkey (type: int)
- mode: hash
+ Select Operator
+ expressions: l_partkey (type: int)
outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ Reducer 4
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -103,38 +88,60 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
+ Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
- 1 l_partkey (type: int)
- outputColumnNames: _col0, _col1, _col3
- input vertices:
- 1 Map 3
- Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col3
- input vertices:
- 1 Map 4
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: lineitem
+ Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col3 (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: vectorized
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Reducer 4
+ Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: int), _col2 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -181,60 +188,80 @@ where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-2 is a root stage
+ Stage-3 is a root stage
+ Stage-2 depends on stages: Stage-3
Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-2
+ Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 3
- Map Operator Tree:
- TableScan
- alias: li
- Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (((l_partkey is not null and l_orderkey is not null) and l_linenumber is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: int)
- 1 l_partkey (type: int)
- Local Work:
- Map Reduce Local Work
- Map 4
+ Map 2
Map Operator Tree:
TableScan
alias: lineitem
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: l_orderkey (type: int), 1 (type: int)
+ expressions: l_orderkey (type: int), l_linenumber (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int), _col4 (type: int)
+ 0 _col0 (type: int), _col3 (type: int)
1 _col0 (type: int), _col1 (type: int)
Local Work:
Map Reduce Local Work
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: lineitem
+ Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((l_linenumber = 1) and l_orderkey is not null) and l_partkey is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col0 (type: int), _col3 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 14 Data size: 1714 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
alias: lineitem
@@ -242,17 +269,21 @@ STAGE PLANS:
Filter Operator
predicate: l_partkey is not null (type: boolean)
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: l_partkey (type: int)
- mode: hash
+ Select Operator
+ expressions: l_partkey (type: int)
outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ Reducer 4
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -265,33 +296,23 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 l_partkey (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
input vertices:
- 1 Map 3
+ 0 Map 1
Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col1 (type: int), _col4 (type: int)
- 1 _col0 (type: int), _col1 (type: int)
- outputColumnNames: _col0, _col3
- input vertices:
- 1 Map 4
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col3 (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 29 Data size: 3627 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Select Operator
+ expressions: _col1 (type: int), _col2 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized
Stage: Stage-0
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
index 3363c8b..6bbb6b9 100644
--- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
@@ -126,11 +126,11 @@ STAGE PLANS:
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: bo (type: boolean), b (type: bigint)
- outputColumnNames: bo, b
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(b)
- keys: bo (type: boolean)
+ aggregations: max(_col1)
+ keys: _col0 (type: boolean)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out b/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
index 9ec8538..b78aec9 100644
--- a/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_string_concat.q.out
@@ -300,11 +300,11 @@ STAGE PLANS:
alias: vectortab2korc
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: dt (type: date)
- outputColumnNames: dt
+ expressions: concat(concat(concat('Quarter ', UDFToString(UDFToInteger(((UDFToDouble((month(dt) - 1)) / 3.0) + 1.0)))), '-'), UDFToString(year(dt))) (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: concat(concat(concat('Quarter ', UDFToString(UDFToInteger((((month(dt) - 1) / 3) + 1)))), '-'), UDFToString(year(dt))) (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 918712 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
index f61ef7d..b2dd09d 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
@@ -129,10 +129,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
- outputColumnNames: ctinyint
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(ctinyint)
+ aggregations: sum(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -435,10 +435,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
- outputColumnNames: cbigint
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(cbigint)
+ aggregations: sum(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -741,10 +741,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float)
- outputColumnNames: cfloat
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(cfloat)
+ aggregations: sum(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
@@ -990,14 +990,14 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cstring2 like '%b%') or ((79.553 <> cint) or (cbigint < cdouble))) (type: boolean)
+ predicate: ((cstring2 like '%b%') or ((79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)
- outputColumnNames: cbigint, cfloat, ctinyint
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cbigint), stddev_pop(cbigint), var_samp(cbigint), count(), sum(cfloat), min(ctinyint)
+ aggregations: avg(_col0), stddev_pop(_col0), var_samp(_col0), count(), sum(_col1), min(_col2)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
@@ -1014,7 +1014,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: double), (- _col0) (type: double), (-6432 + _col0) (type: double), _col1 (type: double), (- (-6432 + _col0)) (type: double), ((- (-6432 + _col0)) + (-6432 + _col0)) (type: double), _col2 (type: double), (- (-6432 + _col0)) (type: double), (-6432 + (- (-6432 + _col0))) (type: double), (- (-6432 + _col0)) (type: double), ((- (-6432 + _col0)) / (- (-6432 + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432 + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
+ expressions: _col0 (type: double), (- _col0) (type: double), (-6432.0 + _col0) (type: double), _col1 (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) + (-6432.0 + _col0)) (type: double), _col2 (type: double), (- (-6432.0 + _col0)) (type: double), (-6432.0 + (- (-6432.0 + _col0))) (type: double), (- (-6432.0 + _col0)) (type: double), ((- (-6432.0 + _col0)) / (- (-6432.0 + _col0))) (type: double), _col3 (type: bigint), _col4 (type: double), (_col2 % _col1) (type: double), (- _col2) (type: double), ((- (-6432.0 + _col0)) * (- _col0)) (type: double), _col5 (type: tinyint), (- _col5) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17
Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
index 41ee583..be00d16 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
@@ -82,15 +82,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cfloat < 3569.0) and ((10.175 >= cdouble) and (cboolean1 <> 1))) or ((ctimestamp1 > 11) and ((ctimestamp2 <> 12) and (ctinyint < 9763215.5639)))) (type: boolean)
+ predicate: (((cfloat < 3569.0) and ((10.175 >= cdouble) and (cboolean1 <> 1))) or ((UDFToDouble(ctimestamp1) > 11.0) and ((UDFToDouble(ctimestamp2) <> 12.0) and (UDFToDouble(ctinyint) < 9763215.5639)))) (type: boolean)
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
- outputColumnNames: cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(ctinyint), sum(cfloat), stddev_pop(cfloat), stddev_pop(ctinyint), max(cfloat), min(ctinyint)
- keys: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
+ aggregations: max(_col1), sum(_col3), stddev_pop(_col3), stddev_pop(_col1), max(_col3), min(_col1)
+ keys: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
@@ -110,7 +110,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * UDFToDouble(_col3)) (type: double), _col7 (type: double), _col8 (type: double), (UDFToDouble(((- _col1) + _col5)) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -336,15 +336,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cfloat < 3569.0) and ((10.175 >= cdouble) and (cboolean1 <> 1))) or ((ctimestamp1 > -1.388) and ((ctimestamp2 <> -1.3359999999999999) and (ctinyint < 9763215.5639)))) (type: boolean)
+ predicate: (((cfloat < 3569.0) and ((10.175 >= cdouble) and (cboolean1 <> 1))) or ((UDFToDouble(ctimestamp1) > -1.388) and ((UDFToDouble(ctimestamp2) <> -1.3359999999999999) and (UDFToDouble(ctinyint) < 9763215.5639)))) (type: boolean)
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
- outputColumnNames: cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(ctinyint), sum(cfloat), stddev_pop(cfloat), stddev_pop(ctinyint), max(cfloat), min(ctinyint)
- keys: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string)
+ aggregations: max(_col1), sum(_col3), stddev_pop(_col3), stddev_pop(_col1), max(_col3), min(_col1)
+ keys: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 2730 Data size: 83809 Basic stats: COMPLETE Column stats: NONE
@@ -364,7 +364,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * UDFToDouble(_col3)) (type: double), _col7 (type: double), _col8 (type: double), (UDFToDouble(((- _col1) + _col5)) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_14.q.out b/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
index 1de639b..cbfa70e 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_14.q.out
@@ -82,15 +82,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((ctinyint <= cbigint) and ((cint <= cdouble) or (ctimestamp2 < ctimestamp1))) and (cdouble < ctinyint)) and ((cbigint > -257) or (cfloat < cint))) (type: boolean)
+ predicate: ((((UDFToLong(ctinyint) <= cbigint) and ((UDFToDouble(cint) <= cdouble) or (ctimestamp2 < ctimestamp1))) and (cdouble < UDFToDouble(ctinyint))) and ((cbigint > -257) or (cfloat < UDFToFloat(cint)))) (type: boolean)
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double)
- outputColumnNames: ctimestamp1, cfloat, cstring1, cboolean1, cdouble
+ expressions: ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double), (- (-26.28 + cdouble)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: stddev_samp((- ((- 26.28) + cdouble))), max(cfloat), stddev_pop(cfloat), count(cfloat), var_pop(cfloat), var_samp(cfloat)
- keys: ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double)
+ aggregations: stddev_samp(_col5), max(_col1), stddev_pop(_col1), count(_col1), var_pop(_col1), var_samp(_col1)
+ keys: _col0 (type: timestamp), _col1 (type: float), _col2 (type: string), _col3 (type: boolean), _col4 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
@@ -110,7 +110,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: timestamp), _col1 (type: float), _col2 (type: string), _col3 (type: boolean), _col4 (type: double), (-26.28 + _col4) (type: double), (- (-26.28 + _col4)) (type: double), _col5 (type: double), (_col1 * -26.28) (type: double), _col6 (type: float), (- _col1) (type: float), (- _col6) (type: float), ((- (-26.28 + _col4)) / 10.175) (type: double), _col7 (type: double), _col8 (type: bigint), (- ((- (-26.28 + _col4)) / 10.175)) (type: double), (-1.389 % _col5) (type: double), (_col1 - _col4) (type: double), _col9 (type: double), (_col9 % 10.175) (type: double), _col10 (type: double), (- (_col1 - _col4)) (type: double)
+ expressions: _col0 (type: timestamp), _col1 (type: float), _col2 (type: string), _col3 (type: boolean), _col4 (type: double), (-26.28 + _col4) (type: double), (- (-26.28 + _col4)) (type: double), _col5 (type: double), (UDFToDouble(_col1) * -26.28) (type: double), _col6 (type: float), (- _col1) (type: float), (- _col6) (type: float), ((- (-26.28 + _col4)) / 10.175) (type: double), _col7 (type: double), _col8 (type: bigint), (- ((- (-26.28 + _col4)) / 10.175)) (type: double), (-1.389 % _col5) (type: double), (UDFToDouble(_col1) - _col4) (type: double), _col9 (type: double), (_col9 % 10.175) (type: double), _col10 (type: double), (- (UDFToDouble(_col1) - _col4)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
index 960a058..a58d941 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
@@ -78,15 +78,15 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cstring2 like '%ss%') or (cstring1 like '10%')) or ((cint >= -75) and ((ctinyint = csmallint) and (cdouble >= -3728)))) (type: boolean)
+ predicate: ((cstring2 like '%ss%') or ((cstring1 like '10%') or ((cint >= -75) and ((UDFToShort(ctinyint) = csmallint) and (cdouble >= -3728.0))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float), cboolean1 (type: boolean), cdouble (type: double), cstring1 (type: string), ctinyint (type: tinyint), cint (type: int), ctimestamp1 (type: timestamp)
- outputColumnNames: cfloat, cboolean1, cdouble, cstring1, ctinyint, cint, ctimestamp1
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: stddev_samp(cfloat), min(cdouble), stddev_samp(ctinyint), var_pop(ctinyint), var_samp(cint), stddev_pop(cint)
- keys: cfloat (type: float), cboolean1 (type: boolean), cdouble (type: double), cstring1 (type: string), ctinyint (type: tinyint), cint (type: int), ctimestamp1 (type: timestamp)
+ aggregations: stddev_samp(_col0), min(_col2), stddev_samp(_col4), var_pop(_col4), var_samp(_col5), stddev_pop(_col5)
+ keys: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
@@ -106,7 +106,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp), _col7 (type: double), (-26.28 - _col5) (type: double), _col8 (type: double), (_col2 * 79.553) (type: double), (33 % _col0) (type: float), _col9 (type: double), _col10 (type: double), (-23 % _col2) (type: double), (- _col4) (type: tinyint), _col11 (type: double), (_col5 - _col0) (type: float), (-23 % _col4) (type: int), (- (-26.28 - _col5)) (type: double), _col12 (type: double)
+ expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp), _col7 (type: double), (-26.28 - UDFToDouble(_col5)) (type: double), _col8 (type: double), (_col2 * 79.553) (type: double), (33.0 % _col0) (type: float), _col9 (type: double), _col10 (type: double), (-23.0 % _col2) (type: double), (- _col4) (type: tinyint), _col11 (type: double), (UDFToFloat(_col5) - _col0) (type: float), (-23 % UDFToInteger(_col4)) (type: int), (- (-26.28 - UDFToDouble(_col5))) (type: double), _col12 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
Statistics: Num rows: 6144 Data size: 188618 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_16.q.out b/ql/src/test/results/clientpositive/spark/vectorization_16.q.out
index 473eaf4..a42c30a 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_16.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_16.q.out
@@ -63,11 +63,11 @@ STAGE PLANS:
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp)
- outputColumnNames: cstring1, cdouble, ctimestamp1
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdouble), stddev_samp(cdouble), min(cdouble)
- keys: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp)
+ aggregations: count(_col1), stddev_samp(_col1), min(_col1)
+ keys: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
@@ -87,7 +87,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 2048 Data size: 62872 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639) (type: double), (- (_col1 - 9763215.5639)) (type: double), _col3 (type: bigint), _col4 (type: double), (- _col4) (type: double), (_col4 * _col3) (type: double), _col5 (type: double), (9763215.5639 / _col1) (type: double), (_col3 / -1.389) (type: double), _col4 (type: double)
+ expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639) (type: double), (- (_col1 - 9763215.5639)) (type: double), _col3 (type: bigint), _col4 (type: double), (- _col4) (type: double), (_col4 * UDFToDouble(_col3)) (type: double), _col5 (type: double), (9763215.5639 / _col1) (type: double), (UDFToDouble(_col3) / -1.389) (type: double), _col4 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 2048 Data size: 62872 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_9.q.out b/ql/src/test/results/clientpositive/spark/vectorization_9.q.out
index 17b35fe..8bbeb51 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_9.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_9.q.out
@@ -59,11 +59,11 @@ STAGE PLANS:
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp)
- outputColumnNames: cstring1, cdouble, ctimestamp1
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(cdouble), stddev_samp(cdouble), min(cdouble)
- keys: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp)
+ aggregations: count(_col1), stddev_samp(_col1), min(_col1)
+ keys: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
@@ -83,7 +83,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 2048 Data size: 62872 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639) (type: double), (- (_col1 - 9763215.5639)) (type: double), _col3 (type: bigint), _col4 (type: double), (- _col4) (type: double), (_col4 * _col3) (type: double), _col5 (type: double), (9763215.5639 / _col1) (type: double), (_col3 / -1.389) (type: double), _col4 (type: double)
+ expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639) (type: double), (- (_col1 - 9763215.5639)) (type: double), _col3 (type: bigint), _col4 (type: double), (- _col4) (type: double), (_col4 * UDFToDouble(_col3)) (type: double), _col5 (type: double), (9763215.5639 / _col1) (type: double), (UDFToDouble(_col3) / -1.389) (type: double), _col4 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
Statistics: Num rows: 2048 Data size: 62872 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
index 30d116f..5887839 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out
@@ -168,7 +168,7 @@ STAGE PLANS:
predicate: ((cbigint > 0) and (cbigint < 100000000)) (type: boolean)
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (cbigint - 988888) (type: bigint), (cdouble / (cbigint - 988888)) (type: double), (1.2 / (cbigint - 988888)) (type: double)
+ expressions: (cbigint - 988888) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888))) (type: double), (1.2 / UDFToDouble((cbigint - 988888))) (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -340,10 +340,10 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cdouble >= -500) and (cdouble < -199)) (type: boolean)
+ predicate: ((cdouble >= -500.0) and (cdouble < -199.0)) (type: boolean)
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (cdouble + 200.0) (type: double), (cbigint / (cdouble + 200.0)) (type: double), ((cdouble + 200.0) / (cdouble + 200.0)) (type: double), (3 / (cdouble + 200.0)) (type: double), (1.2 / (cdouble + 200.0)) (type: double)
+ expressions: (cdouble + 200.0) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0)) (type: double), ((cdouble + 200.0) / (cdouble + 200.0)) (type: double), (3.0 / (cdouble + 200.0)) (type: double), (1.2 / (cdouble + 200.0)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col4, _col5
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out b/ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out
index f92e823..f0852ea 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_part_project.q.out
@@ -67,7 +67,7 @@ STAGE PLANS:
alias: alltypesorc_part
Statistics: Num rows: 200 Data size: 54496 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (cdouble + 2) (type: double)
+ expressions: (cdouble + 2.0) (type: double)
outputColumnNames: _col0
Statistics: Num rows: 200 Data size: 54496 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
index 6e784cd..eb8914b 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_pushdown.q.out
@@ -18,17 +18,17 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesorc
- filterExpr: (cbigint < cdouble) (type: boolean)
+ filterExpr: (UDFToDouble(cbigint) < cdouble) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cbigint < cdouble) (type: boolean)
+ predicate: (UDFToDouble(cbigint) < cdouble) (type: boolean)
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
- outputColumnNames: cbigint
+ outputColumnNames: _col0
Statistics: Num rows: 4096 Data size: 125745 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg(cbigint)
+ aggregations: avg(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
[28/61] [abbrv] hive git commit: HIVE-11001 : HS2 http cookie mode
does not honor doAs url parameter (Thejas Nair ,
reviewed by Hari Sankar Sivarama Subramaniyan )
Posted by xu...@apache.org.
HIVE-11001 : HS2 http cookie mode does not honor doAs url parameter (Thejas Nair , reviewed by Hari Sankar Sivarama Subramaniyan )
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77978f30
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77978f30
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77978f30
Branch: refs/heads/spark
Commit: 77978f301deb231c6e139558ee7fd239276ad8aa
Parents: c338059
Author: Thejas Nair <th...@hortonworks.com>
Authored: Sun Jun 14 17:01:10 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Sun Jun 14 17:01:29 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/session/SessionState.java | 3 +++
.../service/cli/thrift/ThriftHttpServlet.java | 18 ++++++++++++------
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77978f30/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 5067f06..37d856c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -338,6 +338,9 @@ public class SessionState {
public SessionState(HiveConf conf, String userName) {
this.conf = conf;
this.userName = userName;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SessionState user: " + userName);
+ }
isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
ls = new LineageState();
resourceMaps = new ResourceMaps();
http://git-wip-us.apache.org/repos/asf/hive/blob/77978f30/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index e1166d3..56c8cb6 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -134,11 +134,6 @@ public class ThriftHttpServlet extends TServlet {
// For a kerberos setup
if (isKerberosAuthMode(authType)) {
clientUserName = doKerberosAuth(request);
- String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
-
- if (doAsQueryParam != null) {
- SessionManager.setProxyUserName(doAsQueryParam);
- }
}
// For password based authentication
else {
@@ -146,8 +141,16 @@ public class ThriftHttpServlet extends TServlet {
}
}
LOG.debug("Client username: " + clientUserName);
+
// Set the thread local username to be used for doAs if true
SessionManager.setUserName(clientUserName);
+
+ // find proxy user if any from query param
+ String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
+ if (doAsQueryParam != null) {
+ SessionManager.setProxyUserName(doAsQueryParam);
+ }
+
clientIpAddress = request.getRemoteAddr();
LOG.debug("Client IP Address: " + clientIpAddress);
// Set the thread local ip address
@@ -225,7 +228,7 @@ public class ThriftHttpServlet extends TServlet {
LOG.debug("Validated the cookie for user " + userName);
}
return userName;
- }
+ }
}
// No valid HS2 generated cookies found, return null
return null;
@@ -522,6 +525,9 @@ public class ThriftHttpServlet extends TServlet {
}
private static String getDoAsQueryParam(String queryString) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("URL query string:" + queryString);
+ }
if (queryString == null) {
return null;
}
[40/61] [abbrv] hive git commit: HIVE-11018 : Turn on cbo in more q
files (Ashutosh Chauhan via Hari Sankar)
Posted by xu...@apache.org.
HIVE-11018 : Turn on cbo in more q files (Ashutosh Chauhan via Hari Sankar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b86f10d0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b86f10d0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b86f10d0
Branch: refs/heads/spark
Commit: b86f10d0f8189ced0dd803dbcb39d6ba5aebf270
Parents: 87ebf13
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Wed Jun 17 00:38:46 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jun 17 00:38:46 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/ambiguitycheck.q | 2 -
.../clientpositive/groupby_grouping_sets6.q | 1 -
.../clientpositive/hybridgrace_hashjoin_2.q | 5 ++-
.../queries/clientpositive/ql_rewrite_gbtoidx.q | 4 +-
.../queries/clientpositive/selectDistinctStar.q | 2 -
.../clientpositive/udaf_percentile_approx_23.q | 2 -
ql/src/test/queries/clientpositive/union35.q | 2 -
.../clientpositive/groupby_grouping_sets6.q.out | 46 +++++++++-----------
.../udaf_percentile_approx_23.q.out | 10 ++---
9 files changed, 33 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/ambiguitycheck.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/ambiguitycheck.q b/ql/src/test/queries/clientpositive/ambiguitycheck.q
index 3b029e1..2334bc6 100644
--- a/ql/src/test/queries/clientpositive/ambiguitycheck.q
+++ b/ql/src/test/queries/clientpositive/ambiguitycheck.q
@@ -1,5 +1,3 @@
-set hive.cbo.enable=false;
-
-- check cluster/distribute/partitionBy
SELECT * FROM SRC x where x.key = 20 CLUSTER BY (key,value) ;
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q b/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
index a500795..882608b 100644
--- a/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
+++ b/ql/src/test/queries/clientpositive/groupby_grouping_sets6.q
@@ -3,7 +3,6 @@ CREATE TABLE T1(a STRING, b STRING, c STRING) ROW FORMAT DELIMITED FIELDS TERMIN
LOAD DATA LOCAL INPATH '../../data/files/grouping_sets.txt' INTO TABLE T1;
set hive.optimize.ppd = false;
-set hive.cbo.enable = false;
-- This filter is not pushed down
EXPLAIN
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q b/ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q
index dd425f4..b3ee414 100644
--- a/ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q
+++ b/ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q
@@ -149,4 +149,7 @@ JOIN src y1 ON (x.key = y1.key)
JOIN srcpart z2 ON (x.value = z2.value)
JOIN src y2 ON (x.value = y2.value)
WHERE z1.key < 'zzzzzzzz' AND z2.key < 'zzzzzzzzzz'
- AND y1.value < 'zzzzzzzz' AND y2.value < 'zzzzzzzzzz';
\ No newline at end of file
+ AND y1.value < 'zzzzzzzz' AND y2.value < 'zzzzzzzzzz';
+
+
+reset hive.cbo.enable;
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/ql_rewrite_gbtoidx.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/ql_rewrite_gbtoidx.q b/ql/src/test/queries/clientpositive/ql_rewrite_gbtoidx.q
index 59703c7..240e74e 100644
--- a/ql/src/test/queries/clientpositive/ql_rewrite_gbtoidx.q
+++ b/ql/src/test/queries/clientpositive/ql_rewrite_gbtoidx.q
@@ -170,4 +170,6 @@ select key, count(key) from tbl group by key order by key;
set hive.optimize.index.groupby=true;
explain select key, count(key) from tbl group by key order by key;
select key, count(key) from tbl group by key order by key;
-DROP TABLE tbl;
\ No newline at end of file
+DROP TABLE tbl;
+
+reset hive.cbo.enable;
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/selectDistinctStar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/selectDistinctStar.q b/ql/src/test/queries/clientpositive/selectDistinctStar.q
index 300d7e0..c1c1c0b 100644
--- a/ql/src/test/queries/clientpositive/selectDistinctStar.q
+++ b/ql/src/test/queries/clientpositive/selectDistinctStar.q
@@ -50,8 +50,6 @@ on src1.key=src2.key;
select distinct * from (select distinct * from src)src1;
-set hive.cbo.enable=false;
-
explain select distinct src.* from src;
select distinct src.* from src;
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
index 028acdb..3685b1e 100644
--- a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
+++ b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
@@ -85,8 +85,6 @@ select * from t10;
select * from t11;
select * from t12;
-set hive.cbo.enable=false;
-
-- NaN
explain
select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) from bucket;
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/queries/clientpositive/union35.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/union35.q b/ql/src/test/queries/clientpositive/union35.q
index 9445711..c2a311e 100644
--- a/ql/src/test/queries/clientpositive/union35.q
+++ b/ql/src/test/queries/clientpositive/union35.q
@@ -1,5 +1,3 @@
-set hive.cbo.enable=true;
-
select * from (
select * from ( select 1 as id , 'foo' as str_1 from src tablesample(5 rows)) f
union all
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out b/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
index ebb8046..d807d44 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_sets6.q.out
@@ -37,20 +37,23 @@ STAGE PLANS:
TableScan
alias: t1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: a (type: string), b (type: string)
- outputColumnNames: a, b
+ Filter Operator
+ predicate: (UDFToDouble(a) = 5.0) (type: boolean)
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: a (type: string), b (type: string), '0' (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 72 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
- sort order: +++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ Select Operator
+ expressions: '5' (type: string), b (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 2 Data size: 72 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ Statistics: Num rows: 2 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
@@ -58,20 +61,13 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
pruneGroupingSetId: true
- Filter Operator
- predicate: (_col0 = 5) (type: boolean)
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: '5' (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/b86f10d0/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
index 14a6874..588f848 100644
--- a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
+++ b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
@@ -524,11 +524,11 @@ STAGE PLANS:
alias: bucket
Statistics: Num rows: 726 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: double)
- outputColumnNames: key
+ expressions: CASE WHEN ((key < 100.0)) THEN (NaN) ELSE (key) END (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 726 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: percentile_approx(CASE WHEN ((key < 100.0)) THEN (UDFToDouble('NaN')) ELSE (key) END, 0.5)
+ aggregations: percentile_approx(_col0, 0.5)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -586,10 +586,10 @@ STAGE PLANS:
Statistics: Num rows: 726 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: double)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 726 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: percentile_approx(key, 0.5)
+ aggregations: percentile_approx(_col0, 0.5)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
[09/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join18_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join18_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/join18_multi_distinct.q.out
index 5560226..18d79d3 100644
--- a/ql/src/test/results/clientpositive/spark/join18_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/join18_multi_distinct.q.out
@@ -50,11 +50,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(value)
- keys: key (type: string)
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -71,11 +71,11 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT value), count(DISTINCT key)
- keys: key (type: string), value (type: string)
+ aggregations: count(DISTINCT _col1), count(DISTINCT _col0)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join29.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join29.q.out b/ql/src/test/results/clientpositive/spark/join29.q.out
index 0b4284c..9479dd1 100644
--- a/ql/src/test/results/clientpositive/spark/join29.q.out
+++ b/ql/src/test/results/clientpositive/spark/join29.q.out
@@ -36,10 +36,10 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
alias: x
@@ -47,19 +47,23 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 2
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 4
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -77,10 +81,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: y
@@ -88,19 +92,23 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 4
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 2
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -116,12 +124,12 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col3
+ outputColumnNames: _col1, _col2, _col3
input vertices:
- 0 Reducer 2
+ 1 Reducer 4
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), UDFToInteger(_col1) (type: int), UDFToInteger(_col3) (type: int)
+ expressions: _col2 (type: string), UDFToInteger(_col3) (type: int), UDFToInteger(_col1) (type: int)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join31.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join31.q.out b/ql/src/test/results/clientpositive/spark/join31.q.out
index a52a8b6..108a1ea 100644
--- a/ql/src/test/results/clientpositive/spark/join31.q.out
+++ b/ql/src/test/results/clientpositive/spark/join31.q.out
@@ -38,10 +38,10 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 5 <- Map 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 4
Map Operator Tree:
TableScan
alias: x
@@ -49,45 +49,42 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 2
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
+ Reducer 5
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Stage: Stage-1
Spark
Edges:
- Reducer 4 <- Map 3 (GROUP, 2)
- Reducer 5 <- Reducer 4 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: y
@@ -95,41 +92,42 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 4
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Reducer 5
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string)
outputColumnNames: _col0
- input vertices:
- 0 Reducer 2
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
@@ -143,7 +141,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 5
+ Reducer 3
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -195,7 +193,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@dest_j1
-POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ]
+POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, ]
POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
PREHOOK: query: select * from dest_j1
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join34.q.out b/ql/src/test/results/clientpositive/spark/join34.q.out
index 8a1ba8d..09b8b6b 100644
--- a/ql/src/test/results/clientpositive/spark/join34.q.out
+++ b/ql/src/test/results/clientpositive/spark/join34.q.out
@@ -158,7 +158,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 20) (type: boolean)
+ predicate: (UDFToDouble(key) < 20.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -224,12 +224,12 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: x1
+ alias: x
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key > 100) (type: boolean)
+ predicate: (UDFToDouble(key) > 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -291,7 +291,7 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [x1]
+ /src [x]
Map 4
Map Operator Tree:
TableScan
@@ -300,16 +300,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -367,7 +371,7 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: string)
- 1 key (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -452,7 +456,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.FieldSchema(name:value, type:string, comment:default), (src)x1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.FieldSchema(name:value, type:string, comment:default), (src)x.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select * from dest_j1
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join35.q.out b/ql/src/test/results/clientpositive/spark/join35.q.out
index 1ed9c90..e84c860 100644
--- a/ql/src/test/results/clientpositive/spark/join35.q.out
+++ b/ql/src/test/results/clientpositive/spark/join35.q.out
@@ -168,22 +168,26 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key < 20) (type: boolean)
+ predicate: (UDFToDouble(key) < 20.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: bigint)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -236,27 +240,31 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: x1
+ alias: x
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key > 100) (type: boolean)
+ predicate: (UDFToDouble(key) > 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: bigint)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -305,7 +313,7 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [x1]
+ /src [x]
Map 6
Map Operator Tree:
TableScan
@@ -314,16 +322,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -398,7 +410,7 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: string)
- 1 key (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 182 Data size: 1938 Basic stats: COMPLETE Column stats: NONE
Select Operator
@@ -500,7 +512,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x1.null, ]
+POSTHOOK: Lineage: dest_j1.val2 EXPRESSION [(src)x.null, (src)x.null, ]
POSTHOOK: Lineage: dest_j1.value SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select * from dest_j1
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join4.q.out b/ql/src/test/results/clientpositive/spark/join4.q.out
index eddcf07..55b3a18 100644
--- a/ql/src/test/results/clientpositive/spark/join4.q.out
+++ b/ql/src/test/results/clientpositive/spark/join4.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -73,20 +73,20 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -158,8 +158,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join40.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out
index 08695ff..02c11cd 100644
--- a/ql/src/test/results/clientpositive/spark/join40.q.out
+++ b/ql/src/test/results/clientpositive/spark/join40.q.out
@@ -22,10 +22,25 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: src
+ alias: x
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: x
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <= 100) (type: boolean)
+ predicate: (UDFToDouble(key) <= 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -37,38 +52,23 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 3
- Map Operator Tree:
- TableScan
- alias: x
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -677,37 +677,45 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col6 (type: string)
+ expressions: _col2 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -3762,32 +3770,40 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join5.q.out b/ql/src/test/results/clientpositive/spark/join5.q.out
index d645e89..1914ad0 100644
--- a/ql/src/test/results/clientpositive/spark/join5.q.out
+++ b/ql/src/test/results/clientpositive/spark/join5.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -73,33 +73,33 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and (UDFToDouble(key) > 15.0)) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Left Outer Join0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col3 (type: string), UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -158,8 +158,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join6.q.out b/ql/src/test/results/clientpositive/spark/join6.q.out
index 4770770..5f2a85a 100644
--- a/ql/src/test/results/clientpositive/spark/join6.q.out
+++ b/ql/src/test/results/clientpositive/spark/join6.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -73,10 +73,10 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -158,8 +158,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join8.q.out b/ql/src/test/results/clientpositive/spark/join8.q.out
index d242c10..e3fac82 100644
--- a/ql/src/test/results/clientpositive/spark/join8.q.out
+++ b/ql/src/test/results/clientpositive/spark/join8.q.out
@@ -58,7 +58,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 10) and (key < 20)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -73,20 +73,20 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 15) and (key < 25)) and key is not null) (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -102,7 +102,7 @@ STAGE PLANS:
predicate: _col2 is null (type: boolean)
Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), null (type: int), _col3 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -161,8 +161,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join9.q.out b/ql/src/test/results/clientpositive/spark/join9.q.out
index 92f0237..c7440da 100644
--- a/ql/src/test/results/clientpositive/spark/join9.q.out
+++ b/ql/src/test/results/clientpositive/spark/join9.q.out
@@ -94,13 +94,17 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -162,14 +166,18 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -226,12 +234,12 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col8
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col8 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col4 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out b/ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out
index dabdcb8..f34153d 100644
--- a/ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_alt_syntax.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[4][tables = [p1, p2]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: explain select p1.p_name, p2.p_name
from part p1 , part p2
PREHOOK: type: QUERY
@@ -21,19 +21,27 @@ STAGE PLANS:
TableScan
alias: p1
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: p_name (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_name (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: p2
+ alias: p1
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: p_name (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
- value expressions: p_name (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -42,19 +50,15 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col1, _col13
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col13 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join_vc.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_vc.q.out b/ql/src/test/results/clientpositive/spark/join_vc.q.out
index 8b25d4f..27265a6 100644
--- a/ql/src/test/results/clientpositive/spark/join_vc.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_vc.q.out
@@ -157,39 +157,47 @@ STAGE PLANS:
alias: t1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Map 4
Map Operator Tree:
TableScan
- alias: t2
+ alias: t1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: BLOCK__OFFSET__INSIDE__FILE (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col7
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col7 (type: bigint)
+ expressions: _col1 (type: bigint)
outputColumnNames: _col0
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out b/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
index e95d2ab..c6f9039 100644
--- a/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
+++ b/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
@@ -555,10 +555,10 @@ STAGE PLANS:
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: hr (type: string)
- outputColumnNames: hr
+ outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: hr (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
index 1efa9e7..7101beb 100644
--- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
@@ -195,12 +195,12 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string), key (type: string)
- outputColumnNames: value, key
+ expressions: value (type: string), (UDFToDouble(key) + 1.0) (type: double)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum((key + 1))
- keys: value (type: string)
+ aggregations: sum(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -289,12 +289,12 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: value (type: string), key (type: string)
- outputColumnNames: value, key
+ expressions: value (type: string), (UDFToDouble(key) + 1.0) (type: double)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: avg((key + 1))
- keys: value (type: string)
+ aggregations: avg(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -384,10 +384,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double)
- outputColumnNames: cdouble
+ outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: cdouble (type: double)
+ keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
@@ -564,10 +564,10 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cdouble (type: double)
- outputColumnNames: ctinyint, cdouble
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: ctinyint (type: tinyint), cdouble (type: double)
+ keys: _col0 (type: tinyint), _col1 (type: double)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
@@ -780,11 +780,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), key (type: string)
- outputColumnNames: value, key
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(key)
- keys: value (type: string)
+ aggregations: sum(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -883,10 +883,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Map 4 (GROUP, 2)
- Reducer 6 <- Reducer 5 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 6 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -896,11 +896,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -911,18 +911,18 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: bigint)
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -942,18 +942,32 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
- Number of rows: 2
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Number of rows: 3
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.3
+ value expressions: _col0 (type: string), _col1 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 3
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 3
+ Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
@@ -961,19 +975,23 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col3
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 4
+ Select Operator
+ expressions: _col0 (type: string), _col3 (type: bigint), _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Limit
+ Number of rows: 4
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -982,30 +1000,16 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
- Number of rows: 3
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.3
- value expressions: _col0 (type: string), _col1 (type: bigint)
- Reducer 6
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 3
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Number of rows: 2
+ Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Stage: Stage-0
@@ -1039,16 +1043,16 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: value (type: string), key (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: value (type: string)
+ key expressions: _col0 (type: string)
sort order: +
- Map-reduce partition columns: value (type: string)
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
- value expressions: key (type: string)
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -1293,15 +1297,15 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: concat(key, value, value, value, value, value, value, value, value, value) (type: string), key (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: concat(key, value, value, value, value, value, value, value, value, value) (type: string)
+ key expressions: _col0 (type: string)
sort order: +
- Map-reduce partition columns: concat(key, value, value, value, value, value, value, value, value, value) (type: string)
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out
index 3285ad9..4480310 100644
--- a/ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out
+++ b/ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 20) (type: boolean)
+ predicate: (UDFToDouble(key) < 20.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), '22' (type: string)
@@ -92,7 +92,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 20) and (key < 40)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 20.0) and (UDFToDouble(key) < 40.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), '33' (type: string)
[49/61] [abbrv] hive git commit: update schematool
Posted by xu...@apache.org.
update schematool
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/95033570
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/95033570
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/95033570
Branch: refs/heads/spark
Commit: 950335705cc0946dc3c7b2337102556cc86c0ca8
Parents: dd30afc
Author: Chao Sun <su...@apache.org>
Authored: Mon May 11 09:45:55 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Thu Jun 18 12:52:12 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/95033570/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
index a21b6d1..c4062e9 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -48,7 +48,7 @@ public class MetaStoreSchemaInfo {
// that has a corresponding schema. eg "0.13.1" is equivalent to "0.13.0"
private static final Map<String, String> EQUIVALENT_VERSIONS =
ImmutableMap.of("0.13.1", "0.13.0",
- "1.0.0", "0.14.0");
+ "1.0.0", "0.14.0", "1.0.1", "1.0.0", "1.1.1", "1.1.0");
public MetaStoreSchemaInfo(String hiveHome, HiveConf hiveConf, String dbType) throws HiveMetaException {
this.hiveHome = hiveHome;
@@ -156,4 +156,4 @@ public class MetaStoreSchemaInfo {
}
}
-}
\ No newline at end of file
+}
[39/61] [abbrv] hive git commit: HIVE-10974: Use
Configuration::getRaw() for the Base64 data (Gopal V,
reviewed by Sergey Shelukhin)
Posted by xu...@apache.org.
HIVE-10974: Use Configuration::getRaw() for the Base64 data (Gopal V, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/87ebf13f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/87ebf13f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/87ebf13f
Branch: refs/heads/spark
Commit: 87ebf13f974b2e705c79802885152efeabaceeae
Parents: b080e06
Author: Gopal V <go...@apache.org>
Authored: Tue Jun 16 21:02:49 2015 -0700
Committer: Gopal V <go...@apache.org>
Committed: Tue Jun 16 21:02:49 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/87ebf13f/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 5f92d11..024e05c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -404,7 +404,7 @@ public final class Utilities {
LOG.info("local path = " + localPath);
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
LOG.debug("Loading plan from string: "+path.toUri().getPath());
- String planString = conf.get(path.toUri().getPath());
+ String planString = conf.getRaw(path.toUri().getPath());
if (planString == null) {
LOG.info("Could not find plan string in conf");
return null;
[50/61] [abbrv] hive git commit: update MetaStoreSchemaInfo for 1.2.1
Posted by xu...@apache.org.
update MetaStoreSchemaInfo for 1.2.1
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/49d8414d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/49d8414d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/49d8414d
Branch: refs/heads/spark
Commit: 49d8414d0e0f220a8924120c75a4013b942a4612
Parents: 9503357
Author: Thejas Nair <th...@hortonworks.com>
Authored: Thu Jun 18 12:57:19 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Thu Jun 18 12:58:01 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/49d8414d/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
index c4062e9..a5eed6d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -48,7 +48,11 @@ public class MetaStoreSchemaInfo {
// that has a corresponding schema. eg "0.13.1" is equivalent to "0.13.0"
private static final Map<String, String> EQUIVALENT_VERSIONS =
ImmutableMap.of("0.13.1", "0.13.0",
- "1.0.0", "0.14.0", "1.0.1", "1.0.0", "1.1.1", "1.1.0");
+ "1.0.0", "0.14.0",
+ "1.0.1", "1.0.0",
+ "1.1.1", "1.1.0",
+ "1.2.1", "1.2.0"
+ );
public MetaStoreSchemaInfo(String hiveHome, HiveConf hiveConf, String dbType) throws HiveMetaException {
this.hiveHome = hiveHome;
[03/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union28.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union28.q.out b/ql/src/test/results/clientpositive/spark/union28.q.out
index 5160879..98582df 100644
--- a/ql/src/test/results/clientpositive/spark/union28.q.out
+++ b/ql/src/test/results/clientpositive/spark/union28.q.out
@@ -73,20 +73,18 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string), value (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
Map 4
Map Operator Tree:
TableScan
@@ -94,76 +92,56 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string), value (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.union_subq_union
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.union_subq_union
Reducer 5
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.union_subq_union
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.union_subq_union
Stage: Stage-0
Move Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union30.q.out b/ql/src/test/results/clientpositive/spark/union30.q.out
index a81417d..3409623 100644
--- a/ql/src/test/results/clientpositive/spark/union30.q.out
+++ b/ql/src/test/results/clientpositive/spark/union30.q.out
@@ -87,20 +87,18 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string), value (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
Map 4
Map Operator Tree:
TableScan
@@ -108,20 +106,18 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string), value (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
Map 6
Map Operator Tree:
TableScan
@@ -146,59 +142,41 @@ STAGE PLANS:
Reducer 3
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.union_subq_union
+ Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.union_subq_union
Reducer 5
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.union_subq_union
+ Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.union_subq_union
Stage: Stage-0
Move Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union33.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out
index 680a627..0e6b1aa 100644
--- a/ql/src/test/results/clientpositive/spark/union33.q.out
+++ b/ql/src/test/results/clientpositive/spark/union33.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 0) (type: boolean)
+ predicate: (UDFToDouble(key) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: '0' (type: string), value (type: string)
@@ -72,11 +72,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -205,11 +205,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -225,7 +225,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 0) (type: boolean)
+ predicate: (UDFToDouble(key) = 0.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: '0' (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union4.q.out b/ql/src/test/results/clientpositive/spark/union4.q.out
index cdda727..c121ef0 100644
--- a/ql/src/test/results/clientpositive/spark/union4.q.out
+++ b/ql/src/test/results/clientpositive/spark/union4.q.out
@@ -60,7 +60,7 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -148,7 +148,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@tmptable
POSTHOOK: Lineage: tmptable.key EXPRESSION []
-POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src)s2.null, ]
+POSTHOOK: Lineage: tmptable.value EXPRESSION []
PREHOOK: query: select * from tmptable x sort by x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@tmptable
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union5.q.out b/ql/src/test/results/clientpositive/spark/union5.q.out
index a65c4e8..afee988 100644
--- a/ql/src/test/results/clientpositive/spark/union5.q.out
+++ b/ql/src/test/results/clientpositive/spark/union5.q.out
@@ -46,7 +46,7 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
@@ -70,22 +70,18 @@ STAGE PLANS:
expressions: 'tst1' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
@@ -112,22 +108,18 @@ STAGE PLANS:
expressions: 'tst2' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union6.q.out b/ql/src/test/results/clientpositive/spark/union6.q.out
index 8146e12..6176beb 100644
--- a/ql/src/test/results/clientpositive/spark/union6.q.out
+++ b/ql/src/test/results/clientpositive/spark/union6.q.out
@@ -121,7 +121,7 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@tmptable
POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmptable.value EXPRESSION [(src1)s2.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: select * from tmptable x sort by x.key, x.value
PREHOOK: type: QUERY
PREHOOK: Input: default@tmptable
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union7.q.out b/ql/src/test/results/clientpositive/spark/union7.q.out
index 7427959..4a81283 100644
--- a/ql/src/test/results/clientpositive/spark/union7.q.out
+++ b/ql/src/test/results/clientpositive/spark/union7.q.out
@@ -51,22 +51,18 @@ STAGE PLANS:
expressions: key (type: string)
outputColumnNames: _col0
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -78,22 +74,18 @@ STAGE PLANS:
expressions: 'tst1' (type: string)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 279 Basic stats: COMPLETE Column stats: PARTIAL
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: PARTIAL
- value expressions: _col1 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union8.q.out b/ql/src/test/results/clientpositive/spark/union8.q.out
index d7211ed..03545f5 100644
--- a/ql/src/test/results/clientpositive/spark/union8.q.out
+++ b/ql/src/test/results/clientpositive/spark/union8.q.out
@@ -42,7 +42,7 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -58,7 +58,7 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: s3
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union9.q.out b/ql/src/test/results/clientpositive/spark/union9.q.out
index 804ca1e..d420ef1 100644
--- a/ql/src/test/results/clientpositive/spark/union9.q.out
+++ b/ql/src/test/results/clientpositive/spark/union9.q.out
@@ -46,7 +46,7 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: s2
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
@@ -64,7 +64,7 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: s3
+ alias: s1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_1.q.out b/ql/src/test/results/clientpositive/spark/union_remove_1.q.out
index bf0fc20..ba0e293 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_1.q.out
@@ -79,11 +79,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -100,11 +100,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out
index a9d4b51..2718775 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out
@@ -98,7 +98,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(1) (type: bigint)
+ expressions: key (type: string), 1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -116,11 +116,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -136,7 +136,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(2) (type: bigint)
+ expressions: key (type: string), 2 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_15.q.out b/ql/src/test/results/clientpositive/spark/union_remove_15.q.out
index 58abd21..26cfbab 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_15.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_15.q.out
@@ -85,11 +85,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -106,11 +106,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out
index 356e79a..7a7aaf2 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out
@@ -88,11 +88,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -109,11 +109,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_18.q.out b/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
index 2c01a5c..a5e15c5 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
@@ -83,11 +83,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), ds (type: string)
- outputColumnNames: key, ds
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), ds (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -104,11 +104,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), ds (type: string)
- outputColumnNames: key, ds
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), ds (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
index d1e2312..ad44400 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
@@ -83,11 +83,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -104,11 +104,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -273,49 +273,49 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 30 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 7) (type: boolean)
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 7.0) (type: boolean)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: '7' (type: string)
- outputColumnNames: key
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 30 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 7) (type: boolean)
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(key) = 7.0) (type: boolean)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: '7' (type: string)
- outputColumnNames: key
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
@@ -324,23 +324,15 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: '7' (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 14 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 14 Data size: 14 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
@@ -348,23 +340,15 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: '7' (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 14 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 14 Data size: 14 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Stage: Stage-0
Move Operator
@@ -451,43 +435,49 @@ STAGE PLANS:
TableScan
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ Filter Operator
+ predicate: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0) (type: boolean)
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ Filter Operator
+ predicate: ((UDFToDouble(key) + UDFToDouble(key)) >= 7.0) (type: boolean)
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -497,20 +487,17 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ expressions: (UDFToDouble(_col0) + UDFToDouble(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col0 >= 7.0) (type: boolean)
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
@@ -520,20 +507,17 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ expressions: (UDFToDouble(_col0) + UDFToDouble(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col0 >= 7.0) (type: boolean)
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Stage: Stage-0
Move Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_2.q.out b/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
index 59d88cb..26c4eff 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
@@ -84,11 +84,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -104,7 +104,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(1) (type: bigint)
+ expressions: key (type: string), 1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -121,7 +121,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(2) (type: bigint)
+ expressions: key (type: string), 2 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_20.q.out b/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
index f57353f..1d67177 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
@@ -81,11 +81,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -102,11 +102,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_21.q.out b/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
index 48867fb..9f5b070 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
@@ -81,20 +81,18 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
Map 3
Map Operator Tree:
TableScan
@@ -102,68 +100,48 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
Stage: Stage-0
Move Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_22.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_22.q.out b/ql/src/test/results/clientpositive/spark/union_remove_22.q.out
index c41e12f..2e01432 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_22.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_22.q.out
@@ -284,11 +284,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -305,11 +305,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_24.q.out b/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
index e989ed2..2659798 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
@@ -77,11 +77,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -98,11 +98,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
index a5a0126..0a94684 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
@@ -95,11 +95,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -116,11 +116,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
index 7d94d1d..6c3d596 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
@@ -84,11 +84,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -105,11 +105,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
index b31b3e6..55baa8b 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
@@ -91,11 +91,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -111,7 +111,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(1) (type: bigint)
+ expressions: key (type: string), 1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -128,7 +128,7 @@ STAGE PLANS:
alias: inputtbl1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), UDFToLong(2) (type: bigint)
+ expressions: key (type: string), 2 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
index 988973f..c981ae4 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out
@@ -432,11 +432,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -453,11 +453,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/union_remove_7.q.out b/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
index 2a5eccf..084fbd6 100644
--- a/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
+++ b/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
@@ -83,11 +83,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
@@ -104,11 +104,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
[06/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
index 0cbd718..ed536c2 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out
@@ -127,34 +127,35 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: src
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: min(key)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: string)
- auto parallelism: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -203,8 +204,8 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [src]
- Map 4
+ /src [a]
+ Map 3
Map Operator Tree:
TableScan
alias: a
@@ -214,13 +215,24 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Group By Operator
+ aggregations: min(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -271,51 +283,26 @@ STAGE PLANS:
Truncated Path -> Alias:
/src [a]
Reducer 2
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: ((_col2 < 5.0) and _col0 is not null) (type: boolean)
- Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 21 Data size: 223 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: _col3 (type: double), _col4 (type: double)
- auto parallelism: false
- Reducer 3
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col8, _col9
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col2, _col3
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col8 (type: double), _col9 (type: double)
+ expressions: _col0 (type: string), _col2 (type: double), _col3 (type: double)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -331,6 +318,35 @@ STAGE PLANS:
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
+ Reducer 4
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(_col1) + 1.0) < 5.0) (type: boolean)
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), (UDFToDouble(_col1) + 2.0) (type: double), (UDFToDouble(_col1) + 3.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 21 Data size: 222 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 21 Data size: 222 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: double), _col2 (type: double)
+ auto parallelism: false
Stage: Stage-0
Fetch Operator
@@ -501,33 +517,30 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: src
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: min(key)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: string)
+ tag: 0
auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
@@ -577,8 +590,8 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [src]
- Map 4
+ /src [a]
+ Map 3
Map Operator Tree:
TableScan
alias: a
@@ -588,13 +601,24 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Group By Operator
+ aggregations: min(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -645,43 +669,18 @@ STAGE PLANS:
Truncated Path -> Alias:
/src [a]
Reducer 2
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: (_col2 < 5.0) (type: boolean)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: _col3 (type: double), _col4 (type: double)
- auto parallelism: false
- Reducer 3
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col8, _col9
+ outputColumnNames: _col0, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col8 (type: double), _col9 (type: double)
+ expressions: _col0 (type: string), _col2 (type: double), _col3 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -705,6 +704,31 @@ STAGE PLANS:
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
+ Reducer 4
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(_col1) + 1.0) < 5.0) (type: boolean)
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), (UDFToDouble(_col1) + 2.0) (type: double), (UDFToDouble(_col1) + 3.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: double), _col2 (type: double)
+ auto parallelism: false
Stage: Stage-0
Fetch Operator
@@ -875,33 +899,30 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: src
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: min(key)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
@@ -951,24 +972,31 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [src]
- Map 4
+ /src [a]
+ Map 3
Map Operator Tree:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: min(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1019,43 +1047,18 @@ STAGE PLANS:
Truncated Path -> Alias:
/src [a]
Reducer 2
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: ((_col2 < 5.0) and _col0 is not null) (type: boolean)
- Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 42 Data size: 446 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: _col3 (type: double), _col4 (type: double)
- auto parallelism: false
- Reducer 3
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col8, _col9
+ outputColumnNames: _col0, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col8 (type: double), _col9 (type: double)
+ expressions: _col0 (type: string), _col2 (type: double), _col3 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -1079,6 +1082,35 @@ STAGE PLANS:
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
+ Reducer 4
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(_col1) + 1.0) < 5.0) (type: boolean)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), (UDFToDouble(_col1) + 2.0) (type: double), (UDFToDouble(_col1) + 3.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 42 Data size: 445 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 42 Data size: 445 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: double), _col2 (type: double)
+ auto parallelism: false
Stage: Stage-0
Fetch Operator
@@ -1249,33 +1281,30 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Map 3 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: src
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: min(key)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: string)
+ tag: 0
auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
@@ -1325,8 +1354,8 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [src]
- Map 4
+ /src [a]
+ Map 3
Map Operator Tree:
TableScan
alias: a
@@ -1336,13 +1365,24 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Group By Operator
+ aggregations: min(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1393,43 +1433,18 @@ STAGE PLANS:
Truncated Path -> Alias:
/src [a]
Reducer 2
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), (_col1 + 1) (type: double), (_col1 + 2) (type: double), (_col1 + 3) (type: double)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: (_col2 < 5.0) (type: boolean)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: _col3 (type: double), _col4 (type: double)
- auto parallelism: false
- Reducer 3
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
+ 0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col8, _col9
+ outputColumnNames: _col0, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col8 (type: double), _col9 (type: double)
+ expressions: _col0 (type: string), _col2 (type: double), _col3 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -1453,6 +1468,31 @@ STAGE PLANS:
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
+ Reducer 4
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(_col1) + 1.0) < 5.0) (type: boolean)
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), (UDFToDouble(_col1) + 2.0) (type: double), (UDFToDouble(_col1) + 3.0) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: double), _col2 (type: double)
+ auto parallelism: false
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join1.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join1.q.out
index 67827ed..987c653 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join1.q.out
@@ -37,52 +37,56 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((_col0 > 10) and (_col0 < 20)) and (_col5 > 15)) and (_col5 < 25)) (type: boolean)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ predicate: ((UDFToDouble(_col2) > 15.0) and (UDFToDouble(_col2) < 25.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -151,52 +155,56 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col5 > 15) and (_col5 < 25)) (type: boolean)
+ predicate: ((UDFToDouble(_col2) > 15.0) and (UDFToDouble(_col2) < 25.0)) (type: boolean)
Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join2.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join2.q.out
index 5d6a469..eaafd7e 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join2.q.out
@@ -37,52 +37,65 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > '15') and (key < '25')) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ predicate: (((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '15') and (_col0 < '25')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > '15') and (key < '25')) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ predicate: (((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '10') and (_col0 < '20')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((((_col0 > '10') and (_col0 < '20')) and (_col5 > '15')) and (_col5 < '25')) (type: boolean)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -271,52 +284,53 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > '15') and (key < '25')) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ predicate: (((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > '15') and (key < '25')) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ predicate: (((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((_col0 > '10') and (_col0 < '20')) (type: boolean)
- Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
index 6a0654a..2cb969f 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out
@@ -36,47 +36,66 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Filter Operator
+ predicate: (((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '15') and (_col0 < '25')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Filter Operator
+ predicate: (((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '10') and (_col0 < '20')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Outer Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((((_col0 > '10') and (_col0 < '20')) and (_col5 > '15')) and (_col5 < '25')) (type: boolean)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -264,47 +283,54 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Filter Operator
+ predicate: (((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Filter Operator
+ predicate: (((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Outer Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((((_col5 > '15') and (_col5 < '25')) and (_col0 > '10')) and (_col0 < '20')) (type: boolean)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
[46/61] [abbrv] hive git commit: HIVE-11033: BloomFilter index is not
honored by ORC reader (Prasanth Jayachandran reviewed by Gopal V)
Posted by xu...@apache.org.
HIVE-11033: BloomFilter index is not honored by ORC reader (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8ef6e683
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8ef6e683
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8ef6e683
Branch: refs/heads/spark
Commit: 8ef6e6835356989c9a36d2dfdbcfd71c675ced9b
Parents: 9a511eb
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Thu Jun 18 01:08:38 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Thu Jun 18 01:08:38 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java | 4 ++++
.../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 9 ++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8ef6e683/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
index 0776018..db2ca15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
@@ -57,6 +57,10 @@ public class OrcUtils {
ObjectInspector inspector) {
int numFlattenedCols = getFlattenedColumnsCount(inspector);
boolean[] results = new boolean[numFlattenedCols];
+ if ("*".equals(selectedColumns)) {
+ Arrays.fill(results, true);
+ return results;
+ }
if (selectedColumns != null && !selectedColumns.isEmpty()) {
includeColumnsImpl(results, selectedColumns.toLowerCase(), allColumns, inspector);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8ef6e683/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index 58e19cb..beaf231 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -666,7 +666,6 @@ class RecordReaderImpl implements RecordReader {
private final List<PredicateLeaf> sargLeaves;
private final int[] filterColumns;
private final long rowIndexStride;
- private final OrcProto.BloomFilterIndex[] bloomFilterIndices;
// same as the above array, but indices are set to true
private final boolean[] sargColumns;
public SargApplier(SearchArgument sarg, String[] columnNames, long rowIndexStride,
@@ -674,7 +673,6 @@ class RecordReaderImpl implements RecordReader {
this.sarg = sarg;
sargLeaves = sarg.getLeaves();
filterColumns = mapSargColumns(sargLeaves, columnNames, 0);
- bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()];
this.rowIndexStride = rowIndexStride;
// included will not be null, row options will fill the array with trues if null
sargColumns = new boolean[includedCount];
@@ -694,7 +692,8 @@ class RecordReaderImpl implements RecordReader {
* @throws IOException
*/
public boolean[] pickRowGroups(
- StripeInformation stripe, OrcProto.RowIndex[] indexes) throws IOException {
+ StripeInformation stripe, OrcProto.RowIndex[] indexes,
+ OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
long rowsInStripe = stripe.getNumberOfRows();
int groupsInStripe = (int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride);
boolean[] result = new boolean[groupsInStripe]; // TODO: avoid alloc?
@@ -705,7 +704,7 @@ class RecordReaderImpl implements RecordReader {
OrcProto.ColumnStatistics stats =
indexes[filterColumns[pred]].getEntry(rowGroup).getStatistics();
OrcProto.BloomFilter bf = null;
- if (bloomFilterIndices[filterColumns[pred]] != null) {
+ if (bloomFilterIndices != null && bloomFilterIndices[filterColumns[pred]] != null) {
bf = bloomFilterIndices[filterColumns[pred]].getBloomFilter(rowGroup);
}
leafValues[pred] = evaluatePredicateProto(stats, sargLeaves.get(pred), bf);
@@ -749,7 +748,7 @@ class RecordReaderImpl implements RecordReader {
return null;
}
readRowIndex(currentStripe, included, sargApp.sargColumns);
- return sargApp.pickRowGroups(stripes.get(currentStripe), indexes);
+ return sargApp.pickRowGroups(stripes.get(currentStripe), indexes, bloomFilterIndices);
}
private void clearStreams() throws IOException {
[25/61] [abbrv] hive git commit: HIVE-10992:WebHCat should not create
delegation tokens when Kerberos is not enabled
Posted by xu...@apache.org.
HIVE-10992:WebHCat should not create delegation tokens when Kerberos is not enabled
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/051bf0b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/051bf0b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/051bf0b1
Branch: refs/heads/spark
Commit: 051bf0b1785262e29a269b76c8aa798e8e85117c
Parents: 5fd1167
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Sat Jun 13 10:21:02 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Sat Jun 13 10:21:02 2015 -0700
----------------------------------------------------------------------
.../hive/hcatalog/templeton/tool/TempletonControllerJob.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/051bf0b1/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
index 5c7de80..f47feeb 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
@@ -133,9 +133,10 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi
JobClient jc = new JobClient(new JobConf(job.getConfiguration()));
- Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
- job.getCredentials().addToken(new Text("mr token"), mrdt);
-
+ if(UserGroupInformation.isSecurityEnabled()) {
+ Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
+ job.getCredentials().addToken(new Text("mr token"), mrdt);
+ }
String metastoreTokenStrForm = addHMSToken(job, user);
job.submit();
[43/61] [abbrv] hive git commit: HIVE-11023 : Disable directSQL if
datanucleus.identifierFactory = datanucleus2 (Sushanth Sowmyan,
reviewed by Ashutosh Chauhan)
Posted by xu...@apache.org.
HIVE-11023 : Disable directSQL if datanucleus.identifierFactory = datanucleus2 (Sushanth Sowmyan, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/524cd794
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/524cd794
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/524cd794
Branch: refs/heads/spark
Commit: 524cd7948d8c2b16a057e52608deeaa5fb9fa4ac
Parents: 37e82ba
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Wed Jun 17 16:34:07 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Wed Jun 17 16:39:33 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/524cd794/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index ed810d2..1c21c8b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -128,10 +128,18 @@ class MetaStoreDirectSql {
convertMapNullsToEmptyStrings =
HiveConf.getBoolVar(conf, ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS);
- this.isCompatibleDatastore = ensureDbInit() && runTestQuery();
- if (isCompatibleDatastore) {
- LOG.info("Using direct SQL, underlying DB is " + dbType);
+ String jdoIdFactory = HiveConf.getVar(conf, ConfVars.METASTORE_IDENTIFIER_FACTORY);
+ if (! ("datanucleus1".equalsIgnoreCase(jdoIdFactory))){
+ LOG.warn("Underlying metastore does not use 'datanuclues1' for its ORM naming scheme."
+ + " Disabling directSQL as it uses hand-hardcoded SQL with that assumption.");
+ isCompatibleDatastore = false;
+ } else {
+ isCompatibleDatastore = ensureDbInit() && runTestQuery();
+ if (isCompatibleDatastore) {
+ LOG.info("Using direct SQL, underlying DB is " + dbType);
+ }
}
+
isAggregateStatsCacheEnabled = HiveConf.getBoolVar(conf, ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED);
if (isAggregateStatsCacheEnabled) {
aggrStatsCache = AggregateStatsCache.getInstance(conf);
[48/61] [abbrv] hive git commit: HIVE-11040 : Change Derby dependency
version to 10.10.2.0 (Jason Dere, reviewed by Sushanth Sowmyan,
Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-11040 : Change Derby dependency version to 10.10.2.0 (Jason Dere, reviewed by Sushanth Sowmyan, Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dd30afc3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dd30afc3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dd30afc3
Branch: refs/heads/spark
Commit: dd30afc334d007f70b55d75ab5d42bdb910c534b
Parents: 3f8b0ef
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Thu Jun 18 10:26:39 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Thu Jun 18 10:27:23 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dd30afc3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d0fe683..9014d70 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,7 +116,7 @@
<commons-logging.version>1.1.3</commons-logging.version>
<commons-pool.version>1.5.4</commons-pool.version>
<commons-dbcp.version>1.4</commons-dbcp.version>
- <derby.version>10.11.1.1</derby.version>
+ <derby.version>10.10.2.0</derby.version>
<dropwizard.version>3.1.0</dropwizard.version>
<guava.version>14.0.1</guava.version>
<groovy.version>2.1.6</groovy.version>
[37/61] [abbrv] hive git commit: Revert "Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/hive"
Posted by xu...@apache.org.
Revert "Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive"
This reverts commit 179c962210f7499f563c6545cbe9b45fd4a496ad, reversing
changes made to 71c36445b85688fda6c4e7c4cca72d75bd07425f.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ed7b70a5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ed7b70a5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ed7b70a5
Branch: refs/heads/spark
Commit: ed7b70a510890ead5ca954365baedfddd855e814
Parents: 179c962
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jun 15 17:16:53 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jun 15 17:16:53 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/HiveInputFormat.java | 38 +++++++++-----------
.../hadoop/hive/ql/plan/TableScanDesc.java | 20 +----------
2 files changed, 17 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ed7b70a5/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 2ff3951..54e1d4e 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -450,31 +450,25 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return;
}
- String serializedFilterObj = scanDesc.getSerializedFilterObject();
- String serializedFilterExpr = scanDesc.getSerializedFilterExpr();
- boolean hasObj = serializedFilterObj != null, hasExpr = serializedFilterExpr != null;
- if (!hasObj) {
- Serializable filterObject = scanDesc.getFilterObject();
- if (filterObject != null) {
- serializedFilterObj = Utilities.serializeObject(filterObject);
- }
- }
- if (serializedFilterObj != null) {
- jobConf.set(TableScanDesc.FILTER_OBJECT_CONF_STR, serializedFilterObj);
- }
- if (!hasExpr) {
- serializedFilterExpr = Utilities.serializeExpression(filterExpr);
+ Serializable filterObject = scanDesc.getFilterObject();
+ if (filterObject != null) {
+ jobConf.set(
+ TableScanDesc.FILTER_OBJECT_CONF_STR,
+ Utilities.serializeObject(filterObject));
}
+
String filterText = filterExpr.getExprString();
+ String filterExprSerialized = Utilities.serializeExpression(filterExpr);
if (LOG.isDebugEnabled()) {
- LOG.debug("Pushdown initiated with filterText = " + filterText + ", filterExpr = "
- + filterExpr + ", serializedFilterExpr = " + serializedFilterExpr + " ("
- + (hasExpr ? "desc" : "new") + ")" + (serializedFilterObj == null ? "" :
- (", serializedFilterObj = " + serializedFilterObj + " (" + (hasObj ? "desc" : "new")
- + ")")));
- }
- jobConf.set(TableScanDesc.FILTER_TEXT_CONF_STR, filterText);
- jobConf.set(TableScanDesc.FILTER_EXPR_CONF_STR, serializedFilterExpr);
+ LOG.debug("Filter text = " + filterText);
+ LOG.debug("Filter expression = " + filterExprSerialized);
+ }
+ jobConf.set(
+ TableScanDesc.FILTER_TEXT_CONF_STR,
+ filterText);
+ jobConf.set(
+ TableScanDesc.FILTER_EXPR_CONF_STR,
+ filterExprSerialized);
}
protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass,
http://git-wip-us.apache.org/repos/asf/hive/blob/ed7b70a5/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 9e9a2a2..6282380 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.parse.TableSample;
@@ -73,9 +72,6 @@ public class TableScanDesc extends AbstractOperatorDesc {
private ExprNodeGenericFuncDesc filterExpr;
private transient Serializable filterObject;
- private String serializedFilterExpr;
- private String serializedFilterObject;
-
// Both neededColumnIDs and neededColumns should never be null.
// When neededColumnIDs is an empty list,
@@ -105,6 +101,7 @@ public class TableScanDesc extends AbstractOperatorDesc {
private transient final Table tableMetadata;
+
public TableScanDesc() {
this(null, null);
}
@@ -147,11 +144,7 @@ public class TableScanDesc extends AbstractOperatorDesc {
}
public void setFilterExpr(ExprNodeGenericFuncDesc filterExpr) {
- // TODO: we could avoid serialization if it's the same expr. Check?
this.filterExpr = filterExpr;
- if (filterExpr != null) {
- serializedFilterExpr = Utilities.serializeExpression(filterExpr);
- }
}
public Serializable getFilterObject() {
@@ -160,9 +153,6 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setFilterObject(Serializable filterObject) {
this.filterObject = filterObject;
- if (filterObject != null) {
- serializedFilterObject = Utilities.serializeObject(filterObject);
- }
}
public void setNeededColumnIDs(List<Integer> neededColumnIDs) {
@@ -291,12 +281,4 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setTableSample(TableSample tableSample) {
this.tableSample = tableSample;
}
-
- public String getSerializedFilterExpr() {
- return serializedFilterExpr;
- }
-
- public String getSerializedFilterObject() {
- return serializedFilterObject;
- }
}
[19/61] [abbrv] hive git commit: HIVE-10959 : webhcat launcher job
should reconnect to the running child job on task retry (Ivan Mitic via
Thejas Nair)
Posted by xu...@apache.org.
HIVE-10959 : webhcat launcher job should reconnect to the running child job on task retry (Ivan Mitic via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f263fcd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f263fcd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f263fcd
Branch: refs/heads/spark
Commit: 9f263fcdc3ecc825d24c0212ae5286d4f6007082
Parents: 612603f
Author: Thejas Nair <th...@hortonworks.com>
Authored: Thu Jun 11 00:13:42 2015 -0700
Committer: Thejas Nair <th...@hortonworks.com>
Committed: Thu Jun 11 00:14:47 2015 -0700
----------------------------------------------------------------------
.../hive/hcatalog/templeton/AppConfig.java | 2 +
.../hive/hcatalog/templeton/HiveDelegator.java | 15 +-
.../hive/hcatalog/templeton/JarDelegator.java | 8 +-
.../hcatalog/templeton/LauncherDelegator.java | 14 ++
.../hive/hcatalog/templeton/PigDelegator.java | 13 +-
.../apache/hive/hcatalog/templeton/Server.java | 34 ++-
.../hive/hcatalog/templeton/SqoopDelegator.java | 20 +-
.../hcatalog/templeton/StreamingDelegator.java | 3 +-
.../templeton/tool/JobSubmissionConstants.java | 2 +
.../hcatalog/templeton/tool/LaunchMapper.java | 214 ++++++++++++++-----
.../apache/hadoop/mapred/WebHCatJTShim20S.java | 10 +
.../apache/hadoop/mapred/WebHCatJTShim23.java | 18 +-
.../apache/hadoop/hive/shims/HadoopShims.java | 5 +
13 files changed, 273 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
index 2b37e7f..8244274 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
@@ -96,6 +96,7 @@ public class AppConfig extends Configuration {
public static final String EXEC_MAX_PROCS_NAME = "templeton.exec.max-procs";
public static final String EXEC_TIMEOUT_NAME = "templeton.exec.timeout";
public static final String HADOOP_QUEUE_NAME = "templeton.hadoop.queue.name";
+ public static final String ENABLE_JOB_RECONNECT_DEFAULT = "templeton.enable.job.reconnect.default";
public static final String HADOOP_NAME = "templeton.hadoop";
public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir";
public static final String HCAT_NAME = "templeton.hcat";
@@ -306,6 +307,7 @@ public class AppConfig extends Configuration {
public String libJars() { return get(LIB_JARS_NAME); }
public String hadoopQueueName() { return get(HADOOP_QUEUE_NAME); }
+ public String enableJobReconnectDefault() { return get(ENABLE_JOB_RECONNECT_DEFAULT); }
public String clusterHadoop() { return get(HADOOP_NAME); }
public String clusterHcat() { return get(HCAT_NAME); }
public String clusterPython() { return get(PYTHON_NAME); }
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
index 7b09b8a..0ea964f 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
@@ -46,25 +46,28 @@ public class HiveDelegator extends LauncherDelegator {
public EnqueueBean run(String user, Map<String, Object> userArgs,
String execute, String srcFile, List<String> defines,
List<String> hiveArgs, String otherFiles,
- String statusdir, String callback, String completedUrl, boolean enablelog)
+ String statusdir, String callback, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException
{
runAs = user;
List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir,
- completedUrl, enablelog);
+ completedUrl, enablelog, enableJobReconnect);
return enqueueController(user, userArgs, callback, args);
}
private List<String> makeArgs(String execute, String srcFile,
List<String> defines, List<String> hiveArgs, String otherFiles,
- String statusdir, String completedUrl, boolean enablelog)
+ String statusdir, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect)
throws BadParam, IOException, InterruptedException
{
ArrayList<String> args = new ArrayList<String>();
try {
- args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
+ args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl,
+ enablelog, enableJobReconnect));
args.add("--");
TempletonUtils.addCmdForWindows(args);
addHiveMetaStoreTokenArg();
@@ -117,7 +120,7 @@ public class HiveDelegator extends LauncherDelegator {
private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles,
String statusdir, String completedUrl,
- boolean enablelog)
+ boolean enablelog, Boolean enableJobReconnect)
throws URISyntaxException, IOException,
InterruptedException
{
@@ -135,7 +138,7 @@ public class HiveDelegator extends LauncherDelegator {
}
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
- enablelog, JobType.HIVE));
+ enablelog, enableJobReconnect, JobType.HIVE));
if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals(""))
{
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
index e5832a8..10ff2c0 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
@@ -44,13 +44,13 @@ public class JarDelegator extends LauncherDelegator {
List<String> jarArgs, List<String> defines,
String statusdir, String callback,
boolean usesHcatalog, String completedUrl,
- boolean enablelog, JobType jobType)
+ boolean enablelog, Boolean enableJobReconnect, JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(jar, mainClass,
libjars, files, jarArgs, defines,
- statusdir, usesHcatalog, completedUrl, enablelog, jobType);
+ statusdir, usesHcatalog, completedUrl, enablelog, enableJobReconnect, jobType);
return enqueueController(user, userArgs, callback, args);
}
@@ -59,7 +59,7 @@ public class JarDelegator extends LauncherDelegator {
String libjars, String files,
List<String> jarArgs, List<String> defines,
String statusdir, boolean usesHcatalog, String completedUrl,
- boolean enablelog, JobType jobType)
+ boolean enablelog, Boolean enableJobReconnect, JobType jobType)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
try {
@@ -67,7 +67,7 @@ public class JarDelegator extends LauncherDelegator {
allFiles.add(TempletonUtils.hadoopFsFilename(jar, appConf, runAs));
args.addAll(makeLauncherArgs(appConf, statusdir,
- completedUrl, allFiles, enablelog, jobType));
+ completedUrl, allFiles, enablelog, enableJobReconnect, jobType));
args.add("--");
TempletonUtils.addCmdForWindows(args);
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
index 71a3107..82e5cb8 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
@@ -118,6 +118,7 @@ public class LauncherDelegator extends TempletonDelegator {
String completedUrl,
List<String> copyFiles,
boolean enablelog,
+ Boolean enableJobReconnect,
JobType jobType) {
ArrayList<String> args = new ArrayList<String>();
@@ -150,6 +151,19 @@ public class LauncherDelegator extends TempletonDelegator {
addDef(args, TempletonControllerJob.TEMPLETON_JOB_LAUNCH_TIME_NAME,
Long.toString(System.currentTimeMillis()));
+ if (enableJobReconnect == null) {
+ // If enablejobreconnect param was not passed by a user, use a cluster
+ // wide default
+ if (appConf.enableJobReconnectDefault() != null) {
+ enableJobReconnect = Boolean.parseBoolean(appConf.enableJobReconnectDefault());
+ } else {
+ // default is false
+ enableJobReconnect = false;
+ }
+ }
+ addDef(args, TempletonControllerJob.ENABLE_JOB_RECONNECT,
+ Boolean.toString(enableJobReconnect));
+
// Hadoop queue information
addDef(args, "mapred.job.queue.name", appConf.hadoopQueueName());
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
index a07f66a..2679a97 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
@@ -48,13 +48,14 @@ public class PigDelegator extends LauncherDelegator {
String execute, String srcFile,
List<String> pigArgs, String otherFiles,
String statusdir, String callback,
- boolean usesHcatalog, String completedUrl, boolean enablelog)
+ boolean usesHcatalog, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(execute,
srcFile, pigArgs,
- otherFiles, statusdir, usesHcatalog, completedUrl, enablelog);
+ otherFiles, statusdir, usesHcatalog, completedUrl, enablelog, enableJobReconnect);
return enqueueController(user, userArgs, callback, args);
}
@@ -68,6 +69,8 @@ public class PigDelegator extends LauncherDelegator {
* @param usesHcatalog whether the command uses hcatalog/needs to connect
* to hive metastore server
* @param completedUrl call back url
+ * @param enablelog
+ * @param enableJobReconnect
* @return list of arguments
* @throws BadParam
* @throws IOException
@@ -76,7 +79,8 @@ public class PigDelegator extends LauncherDelegator {
private List<String> makeArgs(String execute, String srcFile,
List<String> pigArgs, String otherFiles,
String statusdir, boolean usesHcatalog,
- String completedUrl, boolean enablelog)
+ String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
//check if the REST command specified explicitly to use hcatalog
@@ -93,7 +97,8 @@ public class PigDelegator extends LauncherDelegator {
allFiles.addAll(Arrays.asList(ofs));
}
- args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG));
+ args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog,
+ enableJobReconnect, JobType.PIG));
boolean shipPigTar = appConf.pigArchive() != null && !appConf.pigArchive().equals("");
boolean shipHiveTar = needsMetastoreAccess && appConf.hiveArchive() != null
&& !appConf.hiveArchive().equals("");
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
index 80d9d2c..27b8e38 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
@@ -647,7 +647,8 @@ public class Server {
@FormParam("arg") List<String> args,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
- @FormParam("enablelog") boolean enablelog)
+ @FormParam("enablelog") boolean enablelog,
+ @FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
@@ -671,12 +672,13 @@ public class Server {
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
+ userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
StreamingDelegator d = new StreamingDelegator(appConf);
return d.run(getDoAsUser(), userArgs, inputs, inputreader, output, mapper, reducer, combiner,
fileList, files, defines, cmdenvs, args,
- statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING);
+ statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect, JobType.STREAMING);
}
/**
@@ -699,7 +701,8 @@ public class Server {
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("usehcatalog") boolean usesHcatalog,
- @FormParam("enablelog") boolean enablelog)
+ @FormParam("enablelog") boolean enablelog,
+ @FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
@@ -717,6 +720,7 @@ public class Server {
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
+ userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
@@ -724,7 +728,7 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
jar, mainClass,
libjars, files, args, defines,
- statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, JobType.JAR);
+ statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, enablejobreconnect, JobType.JAR);
}
/**
@@ -747,7 +751,8 @@ public class Server {
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
@FormParam("usehcatalog") boolean usesHcatalog,
- @FormParam("enablelog") boolean enablelog)
+ @FormParam("enablelog") boolean enablelog,
+ @FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
@@ -765,6 +770,7 @@ public class Server {
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
+ userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
@@ -772,7 +778,7 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
execute, srcFile,
pigArgs, otherFiles,
- statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog);
+ statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, enablejobreconnect);
}
/**
@@ -784,6 +790,8 @@ public class Server {
* @param statusdir where the stderr/stdout of templeton controller job goes
* @param callback URL which WebHCat will call when the sqoop job finishes
* @param enablelog whether to collect mapreduce log into statusdir/logs
+ * @param enablejobreconnect whether to reconnect to a running child job on templeton
+ * controller job retry
*/
@POST
@Path("sqoop")
@@ -794,7 +802,8 @@ public class Server {
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
- @FormParam("enablelog") boolean enablelog)
+ @FormParam("enablelog") boolean enablelog,
+ @FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
IOException, InterruptedException {
verifyUser();
@@ -814,9 +823,10 @@ public class Server {
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
+ userArgs.put("enablejobreconnect", enablejobreconnect);
SqoopDelegator d = new SqoopDelegator(appConf);
return d.run(getDoAsUser(), userArgs, command, optionsFile, otherFiles,
- statusdir, callback, getCompletedUrl(), enablelog, libdir);
+ statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect, libdir);
}
/**
@@ -833,6 +843,8 @@ public class Server {
* @param statusdir where the stderr/stdout of templeton controller job goes
* @param callback URL which WebHCat will call when the hive job finishes
* @param enablelog whether to collect mapreduce log into statusdir/logs
+ * @param enablejobreconnect whether to reconnect to a running child job on templeton
+ * controller job retry
*/
@POST
@Path("hive")
@@ -844,7 +856,8 @@ public class Server {
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
- @FormParam("enablelog") boolean enablelog)
+ @FormParam("enablelog") boolean enablelog,
+ @FormParam("enablejobreconnect") Boolean enablejobreconnect)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
@@ -862,12 +875,13 @@ public class Server {
userArgs.put("statusdir", statusdir);
userArgs.put("callback", callback);
userArgs.put("enablelog", Boolean.toString(enablelog));
+ userArgs.put("enablejobreconnect", enablejobreconnect);
checkEnableLogPrerequisite(enablelog, statusdir);
HiveDelegator d = new HiveDelegator(appConf);
return d.run(getDoAsUser(), userArgs, execute, srcFile, defines, hiveArgs, otherFiles,
- statusdir, callback, getCompletedUrl(), enablelog);
+ statusdir, callback, getCompletedUrl(), enablelog, enablejobreconnect);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
index b205bda..9002482 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SqoopDelegator.java
@@ -45,9 +45,10 @@ public class SqoopDelegator extends LauncherDelegator {
}
public EnqueueBean run(String user,
- Map<String, Object> userArgs, String command,
- String optionsFile, String otherFiles, String statusdir,
- String callback, String completedUrl, boolean enablelog, String libdir)
+ Map<String, Object> userArgs, String command,
+ String optionsFile, String otherFiles, String statusdir,
+ String callback, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect, String libdir)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
IOException, InterruptedException
{
@@ -59,17 +60,19 @@ public class SqoopDelegator extends LauncherDelegator {
}
runAs = user;
List<String> args = makeArgs(command, optionsFile, otherFiles, statusdir,
- completedUrl, enablelog, libdir);
+ completedUrl, enablelog, enableJobReconnect, libdir);
return enqueueController(user, userArgs, callback, args);
}
private List<String> makeArgs(String command, String optionsFile, String otherFiles,
- String statusdir, String completedUrl, boolean enablelog, String libdir)
+ String statusdir, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect, String libdir)
throws BadParam, IOException, InterruptedException
{
ArrayList<String> args = new ArrayList<String>();
try {
- args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog, libdir));
+ args.addAll(makeBasicArgs(optionsFile, otherFiles, statusdir, completedUrl, enablelog,
+ enableJobReconnect, libdir));
args.add("--");
TempletonUtils.addCmdForWindows(args);
args.add(appConf.sqoopPath());
@@ -112,7 +115,8 @@ public class SqoopDelegator extends LauncherDelegator {
}
private List<String> makeBasicArgs(String optionsFile, String otherFiles,
- String statusdir, String completedUrl, boolean enablelog, String libdir)
+ String statusdir, String completedUrl, boolean enablelog,
+ Boolean enableJobReconnect, String libdir)
throws URISyntaxException, FileNotFoundException, IOException,
InterruptedException
{
@@ -152,7 +156,7 @@ public class SqoopDelegator extends LauncherDelegator {
}
}
args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles,
- enablelog, JobType.SQOOP));
+ enablelog, enableJobReconnect, JobType.SQOOP));
if(TempletonUtils.isset(appConf.sqoopArchive())) {
args.add("-archives");
args.add(appConf.sqoopArchive());
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
index 622440b..f487d51 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
@@ -48,6 +48,7 @@ public class StreamingDelegator extends LauncherDelegator {
String callback,
String completedUrl,
boolean enableLog,
+ Boolean enableJobReconnect,
JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
@@ -58,7 +59,7 @@ public class StreamingDelegator extends LauncherDelegator {
return d.run(user, userArgs,
appConf.streamingJar(), null,
null, files, args, defines,
- statusdir, callback, false, completedUrl, enableLog, jobType);
+ statusdir, callback, false, completedUrl, enableLog, enableJobReconnect, jobType);
}
private List<String> makeArgs(List<String> inputs,
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
index d3dc3f7..4e4fcd4 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java
@@ -22,6 +22,7 @@ public interface JobSubmissionConstants {
public static final String COPY_NAME = "templeton.copy";
public static final String STATUSDIR_NAME = "templeton.statusdir";
public static final String ENABLE_LOG = "templeton.enablelog";
+ public static final String ENABLE_JOB_RECONNECT = "templeton.enablejobreconnect";
public static final String JOB_TYPE = "templeton.jobtype";
public static final String JAR_ARGS_NAME = "templeton.args";
public static final String TEMPLETON_JOB_LAUNCH_TIME_NAME = "templeton.job.launch.time";
@@ -31,6 +32,7 @@ public interface JobSubmissionConstants {
public static final String EXIT_FNAME = "exit";
public static final int WATCHER_TIMEOUT_SECS = 10;
public static final int KEEP_ALIVE_MSEC = 60 * 1000;
+ public static final int POLL_JOBPROGRESS_MSEC = 30 * 1000;
/**
* A comma-separated list of files to be added to HADOOP_CLASSPATH in
* {@link org.apache.hive.hcatalog.templeton.tool.LaunchMapper}. Used to localize additional
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
index 91fe247..422e75e 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java
@@ -51,6 +51,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -150,13 +151,8 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
env.put(pathVarName, paths);
}
}
- protected Process startJob(Context context, String user, String overrideClasspath)
+ protected Process startJob(Configuration conf, String jobId, String user, String overrideClasspath)
throws IOException, InterruptedException {
- Configuration conf = context.getConfiguration();
-
- // Kill previously launched child MR jobs started by this launcher to prevent having
- // same jobs running side-by-side
- killLauncherChildJobs(conf, context.getJobID().toString());
copyLocal(COPY_NAME, conf);
String[] jarArgs = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME));
@@ -175,7 +171,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary");
handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path");
handleMapReduceJobTag(jarArgsList, JobSubmissionConstants.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER,
- JobSubmissionConstants.MAPREDUCE_JOB_TAGS, context.getJobID().toString());
+ JobSubmissionConstants.MAPREDUCE_JOB_TAGS, jobId);
return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env);
}
@@ -289,17 +285,105 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
}
}
+ /**
+ * Checks if reconnection to an already running job is enabled and supported for a given
+ * job type.
+ */
+ private boolean reconnectToRunningJobEnabledAndSupported(Configuration conf,
+ LauncherDelegator.JobType jobType) {
+ if (conf.get(ENABLE_JOB_RECONNECT) == null) {
+ return false;
+ }
+
+ Boolean enableJobReconnect = Boolean.parseBoolean(conf.get(ENABLE_JOB_RECONNECT));
+ if (!enableJobReconnect) {
+ return false;
+ }
+
+ // Reconnect is only supported for MR and Streaming jobs at this time
+ return jobType.equals(LauncherDelegator.JobType.JAR)
+ || jobType.equals(LauncherDelegator.JobType.STREAMING);
+ }
+
+ /**
+ * Attempts to reconnect to an already running child job of the templeton launcher. This
+ * is used in cases where the templeton launcher task has failed and is retried by the
+ * MR framework. If reconnect to the child job is possible, the method will continue
+ * tracking its progress until completion.
+ * @return Returns true if reconnect was successful, false if not supported or
+ * no child jobs were found.
+ */
+ private boolean tryReconnectToRunningJob(Configuration conf, Context context,
+ LauncherDelegator.JobType jobType, String statusdir) throws IOException, InterruptedException {
+ if (!reconnectToRunningJobEnabledAndSupported(conf, jobType)) {
+ return false;
+ }
+
+ long startTime = getTempletonLaunchTime(conf);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ WebHCatJTShim tracker = ShimLoader.getHadoopShims().getWebHCatShim(conf, ugi);
+ try {
+ Set<String> childJobs = tracker.getJobs(context.getJobID().toString(), startTime);
+ if (childJobs.size() == 0) {
+ LOG.info("No child jobs found to reconnect with");
+ return false;
+ }
+
+ if (childJobs.size() > 1) {
+ LOG.warn(String.format(
+ "Found more than one child job to reconnect with: %s, skipping reconnect",
+ Arrays.toString(childJobs.toArray())));
+ return false;
+ }
+
+ String childJobIdString = childJobs.iterator().next();
+ org.apache.hadoop.mapred.JobID childJobId =
+ org.apache.hadoop.mapred.JobID.forName(childJobIdString);
+ LOG.info(String.format("Reconnecting to an existing job %s", childJobIdString));
+
+ // Update job state with the childJob id
+ updateJobStatePercentAndChildId(conf, context.getJobID().toString(), null, childJobIdString);
+
+ do {
+ org.apache.hadoop.mapred.JobStatus jobStatus = tracker.getJobStatus(childJobId);
+ if (jobStatus.isJobComplete()) {
+ LOG.info(String.format("Child job %s completed", childJobIdString));
+ int exitCode = 0;
+ if (jobStatus.getRunState() != org.apache.hadoop.mapred.JobStatus.SUCCEEDED) {
+ exitCode = 1;
+ }
+ updateJobStateToDoneAndWriteExitValue(conf, statusdir, context.getJobID().toString(),
+ exitCode);
+ break;
+ }
+
+ String percent = String.format("map %s%%, reduce %s%%",
+ jobStatus.mapProgress()*100, jobStatus.reduceProgress()*100);
+ updateJobStatePercentAndChildId(conf, context.getJobID().toString(), percent, null);
+
+ LOG.info("KeepAlive Heart beat");
+
+ context.progress();
+ Thread.sleep(POLL_JOBPROGRESS_MSEC);
+ } while (true);
+
+ // Reconnect was successful
+ return true;
+ }
+ catch (IOException ex) {
+ LOG.error("Exception encountered in tryReconnectToRunningJob", ex);
+ throw ex;
+ } finally {
+ tracker.close();
+ }
+ }
+
@Override
public void run(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
-
- Process proc = startJob(context,
- conf.get("user.name"),
- conf.get(OVERRIDE_CLASSPATH));
-
+ LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
String statusdir = conf.get(STATUSDIR_NAME);
-
if (statusdir != null) {
try {
statusdir = TempletonUtils.addUserHomeDirectoryIfApplicable(statusdir,
@@ -311,8 +395,20 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
}
}
- Boolean enablelog = Boolean.parseBoolean(conf.get(ENABLE_LOG));
- LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE));
+ // Try to reconnect to a child job if one is found
+ if (tryReconnectToRunningJob(conf, context, jobType, statusdir)) {
+ return;
+ }
+
+ // Kill previously launched child MR jobs started by this launcher to prevent having
+ // same jobs running side-by-side
+ killLauncherChildJobs(conf, context.getJobID().toString());
+
+ // Start the job
+ Process proc = startJob(conf,
+ context.getJobID().toString(),
+ conf.get("user.name"),
+ conf.get(OVERRIDE_CLASSPATH));
ExecutorService pool = Executors.newCachedThreadPool();
executeWatcher(pool, conf, context.getJobID(),
@@ -328,27 +424,67 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
pool.shutdownNow();
}
- writeExitValue(conf, proc.exitValue(), statusdir);
- JobState state = new JobState(context.getJobID().toString(), conf);
- state.setExitValue(proc.exitValue());
- state.setCompleteStatus("done");
- state.close();
+ updateJobStateToDoneAndWriteExitValue(conf, statusdir, context.getJobID().toString(),
+ proc.exitValue());
+ Boolean enablelog = Boolean.parseBoolean(conf.get(ENABLE_LOG));
if (enablelog && TempletonUtils.isset(statusdir)) {
LOG.info("templeton: collecting logs for " + context.getJobID().toString()
- + " to " + statusdir + "/logs");
+ + " to " + statusdir + "/logs");
LogRetriever logRetriever = new LogRetriever(statusdir, jobType, conf);
logRetriever.run();
}
+ }
- if (proc.exitValue() != 0) {
+ private void updateJobStateToDoneAndWriteExitValue(Configuration conf,
+ String statusdir, String jobId, int exitCode) throws IOException {
+ writeExitValue(conf, exitCode, statusdir);
+ JobState state = new JobState(jobId, conf);
+ state.setExitValue(exitCode);
+ state.setCompleteStatus("done");
+ state.close();
+
+ if (exitCode != 0) {
LOG.info("templeton: job failed with exit code "
- + proc.exitValue());
+ + exitCode);
} else {
LOG.info("templeton: job completed with exit code 0");
}
}
+ /**
+ * Updates the job state percent and childid in the templeton storage. Update only
+ * takes place for non-null values.
+ */
+ private static void updateJobStatePercentAndChildId(Configuration conf,
+ String jobId, String percent, String childid) {
+ JobState state = null;
+ try {
+ if (percent != null || childid != null) {
+ state = new JobState(jobId, conf);
+ if (percent != null) {
+ state.setPercentComplete(percent);
+ }
+ if (childid != null) {
+ JobState childState = new JobState(childid, conf);
+ childState.setParent(jobId);
+ state.addChild(childid);
+ state.close();
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("templeton: state error: ", e);
+ } finally {
+ if (state != null) {
+ try {
+ state.close();
+ } catch (IOException e) {
+ LOG.warn(e);
+ }
+ }
+ }
+ }
+
private void executeWatcher(ExecutorService pool, Configuration conf, JobID jobid, InputStream in,
String statusdir, String name) throws IOException {
Watcher w = new Watcher(conf, jobid, in, statusdir, name);
@@ -371,6 +507,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
PrintWriter writer = new PrintWriter(out);
writer.println(exitValue);
writer.close();
+ LOG.info("templeton: Exit value successfully written");
}
}
@@ -414,34 +551,9 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text>
String line;
while ((line = reader.readLine()) != null) {
writer.println(line);
- JobState state = null;
- try {
- String percent = TempletonUtils.extractPercentComplete(line);
- String childid = TempletonUtils.extractChildJobId(line);
-
- if (percent != null || childid != null) {
- state = new JobState(jobid.toString(), conf);
- if (percent != null) {
- state.setPercentComplete(percent);
- }
- if (childid != null) {
- JobState childState = new JobState(childid, conf);
- childState.setParent(jobid.toString());
- state.addChild(childid);
- state.close();
- }
- }
- } catch (IOException e) {
- LOG.error("templeton: state error: ", e);
- } finally {
- if (state != null) {
- try {
- state.close();
- } catch (IOException e) {
- LOG.warn(e);
- }
- }
- }
+ String percent = TempletonUtils.extractPercentComplete(line);
+ String childid = TempletonUtils.extractChildJobId(line);
+ updateJobStatePercentAndChildId(conf, jobid.toString(), percent, childid);
}
writer.flush();
if(out != System.err && out != System.out) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/shims/0.20S/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim20S.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim20S.java b/shims/0.20S/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim20S.java
index 8b165f3..367ea60 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim20S.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim20S.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.util.HashSet;
+import java.util.Set;
/**
* This is in org.apache.hadoop.mapred package because it relies on
@@ -109,5 +111,13 @@ public class WebHCatJTShim20S implements WebHCatJTShim {
public void killJobs(String tag, long timestamp) {
return;
}
+ /**
+ * Get jobs is only supported on hadoop 2.0+.
+ */
+ @Override
+ public Set<String> getJobs(String tag, long timestamp)
+ {
+ return new HashSet<String>();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java b/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
index dd27cce..c85a739 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java
@@ -21,9 +21,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim;
-
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -180,6 +180,22 @@ public class WebHCatJTShim23 implements WebHCatJTShim {
}
/**
+ * Returns all jobs tagged with the given tag that have been started after the
+ * given timestamp. Returned jobIds are MapReduce JobIds.
+ */
+ @Override
+ public Set<String> getJobs(String tag, long timestamp) {
+ Set<ApplicationId> childYarnJobs = getYarnChildJobs(tag, timestamp);
+ Set<String> childJobs = new HashSet<String>();
+ for(ApplicationId id : childYarnJobs) {
+ // Convert to a MapReduce job id
+ String childJobId = TypeConverter.fromYarn(id).toString();
+ childJobs.add(childJobId);
+ }
+ return childJobs;
+ }
+
+ /**
* Queries RM for the list of applications with the given tag that have started
* after the given timestamp.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/9f263fcd/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 5b7e7f6..74785e5 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -393,6 +393,11 @@ public interface HadoopShims {
* given timestamp.
*/
public void killJobs(String tag, long timestamp);
+ /**
+ * Returns all jobs tagged with the given tag that have been started after the
+ * given timestamp. Returned jobIds are MapReduce JobIds.
+ */
+ public Set<String> getJobs(String tag, long timestamp);
}
/**
[59/61] [abbrv] hive git commit: HIVE-11050:
testCliDriver_vector_outer_join.* failures in Unit tests due to unstable data
creation queries (Matt McCline reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-11050: testCliDriver_vector_outer_join.* failures in Unit tests due to unstable data creation queries (Matt McCline reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b97303cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b97303cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b97303cd
Branch: refs/heads/spark
Commit: b97303cdd44e70dfbf6edc557a253c37416874d9
Parents: 5d7bdad
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Jun 19 15:53:36 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Jun 19 15:53:36 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/vector_outer_join1.q | 20 +-
.../queries/clientpositive/vector_outer_join2.q | 19 +-
.../queries/clientpositive/vector_outer_join3.q | 19 +-
.../queries/clientpositive/vector_outer_join4.q | 16 +-
.../clientpositive/tez/vector_outer_join1.q.out | 254 ++++-
.../clientpositive/tez/vector_outer_join2.q.out | 124 +-
.../clientpositive/tez/vector_outer_join3.q.out | 166 ++-
.../clientpositive/tez/vector_outer_join4.q.out | 1060 ++++++++++--------
.../clientpositive/vector_outer_join1.q.out | 246 +++-
.../clientpositive/vector_outer_join2.q.out | 120 +-
.../clientpositive/vector_outer_join3.q.out | 154 ++-
.../clientpositive/vector_outer_join4.q.out | 1052 +++++++++--------
12 files changed, 2055 insertions(+), 1195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/queries/clientpositive/vector_outer_join1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join1.q b/ql/src/test/queries/clientpositive/vector_outer_join1.q
index b695b08..566b67a 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join1.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join1.q
@@ -1,10 +1,17 @@
SET hive.vectorized.execution.enabled=true;
SET hive.auto.convert.join=true;
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
-create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5;
-create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5;
+-- Using cint and ctinyint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+
+select * from small_alltypesorc1a;
+select * from small_alltypesorc2a;
+select * from small_alltypesorc3a;
+select * from small_alltypesorc4a;
create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
@@ -13,12 +20,13 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1;
+ select * from (select * from small_alltypesorc4a) sq4) q;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
+select * from small_alltypesorc_a;
+
explain
select *
from small_alltypesorc_a c
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/queries/clientpositive/vector_outer_join2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join2.q b/ql/src/test/queries/clientpositive/vector_outer_join2.q
index d7abb21..468034e 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join2.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join2.q
@@ -2,10 +2,16 @@ SET hive.vectorized.execution.enabled=true;
SET hive.auto.convert.join=true;
SET hive.vectorized.execution.mapjoin.native.enabled=true;
-create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5;
-create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5;
+-- Using cint and cbigint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc2a as select * from alltypesorc where cint is null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc4a as select * from alltypesorc where cint is null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+
+select * from small_alltypesorc1a;
+select * from small_alltypesorc2a;
+select * from small_alltypesorc3a;
+select * from small_alltypesorc4a;
create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
@@ -14,12 +20,13 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1;
+ select * from (select * from small_alltypesorc4a) sq4) q;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
+select * from small_alltypesorc_a;
+
explain
select count(*), sum(t1.c_cbigint) from (select c.cbigint as c_cbigint
from small_alltypesorc_a c
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/queries/clientpositive/vector_outer_join3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join3.q b/ql/src/test/queries/clientpositive/vector_outer_join3.q
index f256e39..30728c8 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join3.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join3.q
@@ -2,10 +2,16 @@ SET hive.vectorized.execution.enabled=true;
SET hive.auto.convert.join=true;
SET hive.vectorized.execution.mapjoin.native.enabled=true;
-create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5;
-create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5;
-create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5;
+-- Using cint and cstring1 in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc2a as select * from alltypesorc where cint is null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+create table small_alltypesorc4a as select * from alltypesorc where cint is null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5;
+
+select * from small_alltypesorc1a;
+select * from small_alltypesorc2a;
+select * from small_alltypesorc3a;
+select * from small_alltypesorc4a;
create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
@@ -14,13 +20,12 @@ create table small_alltypesorc_a stored as orc as select * from
union all
select * from (select * from small_alltypesorc3a) sq3
union all
- select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1;
-
+ select * from (select * from small_alltypesorc4a) sq4) q;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS;
+select * from small_alltypesorc_a;
explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/queries/clientpositive/vector_outer_join4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join4.q b/ql/src/test/queries/clientpositive/vector_outer_join4.q
index fb9e6e4..6976285 100644
--- a/ql/src/test/queries/clientpositive/vector_outer_join4.q
+++ b/ql/src/test/queries/clientpositive/vector_outer_join4.q
@@ -2,10 +2,16 @@ SET hive.vectorized.execution.enabled=true;
SET hive.auto.convert.join=true;
SET hive.vectorized.execution.mapjoin.native.enabled=true;
-create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null limit 10;
-create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null limit 10;
-create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null limit 10;
-create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null limit 10;
+-- Using cint and ctinyint in test queries
+create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10;
+create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10;
+create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10;
+create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10;
+
+select * from small_alltypesorc1b;
+select * from small_alltypesorc2b;
+select * from small_alltypesorc3b;
+select * from small_alltypesorc4b;
create table small_alltypesorc_b stored as orc as select * from
(select * from (select * from small_alltypesorc1b) sq1
@@ -19,6 +25,8 @@ create table small_alltypesorc_b stored as orc as select * from
ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS;
ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS;
+select * from small_alltypesorc_b;
+
explain
select *
from small_alltypesorc_b c
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
index 95ed151..664f74a 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out
@@ -1,43 +1,92 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +95,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +110,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +133,29 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select *
from small_alltypesorc_a c
@@ -113,11 +183,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -127,11 +197,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
input vertices:
1 Map 2
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3831 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3831 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -141,16 +211,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
Execution mode: vectorized
@@ -178,21 +248,25 @@ left outer join small_alltypesorc_a cd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
--51 NULL -1071480828 -1401575336 -51.0 NULL aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true -51 NULL -1071480828 -1401575336 -51.0 NULL aw724t8c5558x2xneC624 4uE7l74tESBiKfu7c8wM7GA 1969-12-31 16:00:08.451 NULL true true
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL -64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL -64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL -64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL -64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL -64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--64 15601 NULL 1421812187 -64.0 15601.0 NULL 12AEw 1969-12-31 16:00:05.997 1969-12-31 15:59:58.456 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--64 15601 NULL 1960950366 -64.0 15601.0 NULL c0ODOF64 1969-12-31 16:00:13.274 1969-12-31 15:59:58.456 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-11 NULL -1069736047 -453772520 11.0 NULL k17Am8uPHWk02cEf1jet qrXLLNX1 1969-12-31 16:00:02.351 NULL true true 11 NULL -1069736047 -453772520 11.0 NULL k17Am8uPHWk02cEf1jet qrXLLNX1 1969-12-31 16:00:02.351 NULL true true
-11 NULL -1072910839 2048385991 11.0 NULL 0iqrc5 KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false 11 NULL -1072910839 2048385991 11.0 NULL 0iqrc5 KbaDXiN85adbHRx58v 1969-12-31 16:00:02.351 NULL false false
-11 NULL -1073279343 -1595604468 11.0 NULL oj1YrV5Wa P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true 11 NULL -1073279343 -1595604468 11.0 NULL oj1YrV5Wa P76636jJ6qM17d7DIy 1969-12-31 16:00:02.351 NULL true true
-8 NULL -1071363017 1349676361 8.0 NULL Anj0oF IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true 8 NULL -1071363017 1349676361 8.0 NULL Anj0oF IwE1G7Qb0B1NEfV030g 1969-12-31 16:00:15.892 NULL true true
-NULL -5470 -1072076362 1864027286 NULL -5470.0 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true NULL -5470 -1072076362 1864027286 NULL -5470.0 2uLyD28144vklju213J1mr 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:01.836 true true
-NULL -7382 -1073051226 -1887561756 NULL -7382.0 A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false NULL -7382 -1073051226 -1887561756 NULL -7382.0 A34p7oRr2WvUJNf 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:10.331 false false
-NULL -741 -1070883071 -1645852809 NULL -741.0 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false NULL -741 -1070883071 -1645852809 NULL -741.0 0ruyd6Y50JpdGRf6HqD xH7445Rals48VOulSyR5F NULL 1969-12-31 15:59:51.293 false false
-NULL -947 -1070551679 1864027286 NULL -947.0 iUR3Q 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:08.058 false true NULL -947 -1070551679 1864027286 NULL -947.0 iUR3Q 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:08.058 false true
-NULL 8373 -1072081801 1864027286 NULL 8373.0 dPkN74F7 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true NULL 8373 -1072081801 1864027286 NULL 8373.0 dPkN74F7 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:56.465 true true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL -64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL -64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL -64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL -64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select c.ctinyint
from small_alltypesorc_a c
@@ -220,11 +294,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -234,11 +308,11 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 2
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3831 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3831 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -248,16 +322,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Stage: Stage-0
@@ -284,7 +358,6 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
--51
-64
-64
-64
@@ -310,16 +383,81 @@ POSTHOOK: Input: default@small_alltypesorc_a
-64
-64
-64
-11
-11
-11
-11
-11
-11
-11
-11
-11
-8
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
+-64
NULL
NULL
NULL
@@ -359,11 +497,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -373,7 +511,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16 Data size: 3831 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -384,7 +522,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17 Data size: 4214 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count(), sum(_col0)
@@ -400,31 +538,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -472,4 +610,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-41 -1544
+145 -8960
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
index 31ef469..f9159eb 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join2.q.out
@@ -1,43 +1,97 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and cbigint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and cbigint in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cbigint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cbigint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+NULL -13166 626923679 NULL NULL -13166.0 821UdmGbkEf4j NULL 1969-12-31 15:59:55.089 1969-12-31 16:00:15.69 true NULL
+NULL -14426 626923679 NULL NULL -14426.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.505 1969-12-31 16:00:13.309 true NULL
+NULL -14847 626923679 NULL NULL -14847.0 821UdmGbkEf4j NULL 1969-12-31 16:00:00.612 1969-12-31 15:59:43.704 true NULL
+NULL -15632 528534767 NULL NULL -15632.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:53.593 true NULL
+NULL -15830 253665376 NULL NULL -15830.0 1cGVWH7n1QU NULL 1969-12-31 16:00:02.582 1969-12-31 16:00:00.518 true NULL
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+-60 -200 NULL NULL -60.0 -200.0 NULL NULL 1969-12-31 16:00:11.996 1969-12-31 15:59:55.451 NULL NULL
+-61 -7196 NULL NULL -61.0 -7196.0 NULL 8Mlns2Tl6E0g 1969-12-31 15:59:44.823 1969-12-31 15:59:58.174 NULL false
+-61 -7196 NULL NULL -61.0 -7196.0 NULL fUJIN 1969-12-31 16:00:11.842 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL jf1Cw6qhkNToQuud 1969-12-31 16:00:12.388 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL yLiOchx5PfDTFdcMduBTg 1969-12-31 16:00:02.373 1969-12-31 15:59:58.174 NULL false
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +100,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +115,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +138,34 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-60 -200 NULL NULL -60.0 -200.0 NULL NULL 1969-12-31 16:00:11.996 1969-12-31 15:59:55.451 NULL NULL
+-61 -7196 NULL NULL -61.0 -7196.0 NULL 8Mlns2Tl6E0g 1969-12-31 15:59:44.823 1969-12-31 15:59:58.174 NULL false
+-61 -7196 NULL NULL -61.0 -7196.0 NULL fUJIN 1969-12-31 16:00:11.842 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL jf1Cw6qhkNToQuud 1969-12-31 16:00:12.388 1969-12-31 15:59:58.174 NULL false
+-62 -7196 NULL NULL -62.0 -7196.0 NULL yLiOchx5PfDTFdcMduBTg 1969-12-31 16:00:02.373 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+NULL -13166 626923679 NULL NULL -13166.0 821UdmGbkEf4j NULL 1969-12-31 15:59:55.089 1969-12-31 16:00:15.69 true NULL
+NULL -14426 626923679 NULL NULL -14426.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.505 1969-12-31 16:00:13.309 true NULL
+NULL -14847 626923679 NULL NULL -14847.0 821UdmGbkEf4j NULL 1969-12-31 16:00:00.612 1969-12-31 15:59:43.704 true NULL
+NULL -15632 528534767 NULL NULL -15632.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:53.593 true NULL
+NULL -15830 253665376 NULL NULL -15830.0 1cGVWH7n1QU NULL 1969-12-31 16:00:02.582 1969-12-31 16:00:00.518 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select count(*), sum(t1.c_cbigint) from (select c.cbigint as c_cbigint
from small_alltypesorc_a c
@@ -120,11 +200,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -134,7 +214,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 4874 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -145,12 +225,12 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5361 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Select Operator
expressions: _col1 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5361 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(), sum(_col0)
mode: hash
@@ -165,31 +245,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4431 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -237,4 +317,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21 17110062350
+34 -26289186744
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
index 75c4215..2a47afa 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join3.q.out
@@ -1,43 +1,97 @@
-PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: -- Using cint and cstring1 in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1a
-POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: -- Using cint and cstring1 in test queries
+create table small_alltypesorc1a as select * from alltypesorc where cint is not null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1a
-PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2a
-POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and ctinyint is not null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc2a as select * from alltypesorc where cint is null and cstring1 is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2a
-PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3a
-POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc3a as select * from alltypesorc where cint is not null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3a
-PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+PREHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4a
-POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and ctinyint is null order by cint, ctinyint limit 5
+POSTHOOK: query: create table small_alltypesorc4a as select * from alltypesorc where cint is null and cstring1 is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4a
+PREHOOK: query: select * from small_alltypesorc1a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1a
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc2a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2a
+#### A masked pattern was here ####
+-51 NULL NULL -1731061911 -51.0 NULL Pw53BBJ yL443x2437PO5Hv1U3lCjq2D 1969-12-31 16:00:08.451 NULL true false
+-51 NULL NULL -1846191223 -51.0 NULL Ul085f84S33Xd32u x1JC58g0Ukp 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1874052220 -51.0 NULL c61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULL false false
+-51 NULL NULL -1927203921 -51.0 NULL 45ja5suO 42S0I0 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1970551565 -51.0 NULL r2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULL false false
+PREHOOK: query: select * from small_alltypesorc3a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3a
+#### A masked pattern was here ####
+-51 NULL -31312632 1086455747 -51.0 NULL NULL Bc7xt12568c451o64LF5 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -337975743 608681041 -51.0 NULL NULL Ih2r28o6 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -413196097 -306198070 -51.0 NULL NULL F53QcSDPpxYF1Ub 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL -591488718 803603078 -51.0 NULL NULL X616UtmmA3FHan 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -738306196 -460430946 -51.0 NULL NULL dBOqv 1969-12-31 16:00:08.451 NULL NULL false
+PREHOOK: query: select * from small_alltypesorc4a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4a
+#### A masked pattern was here ####
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
(select * from (select * from small_alltypesorc1a) sq1
union all
@@ -46,7 +100,6 @@ PREHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@small_alltypesorc1a
PREHOOK: Input: default@small_alltypesorc2a
@@ -62,7 +115,6 @@ POSTHOOK: query: create table small_alltypesorc_a stored as orc as select * from
select * from (select * from small_alltypesorc3a) sq3
union all
select * from (select * from small_alltypesorc4a) sq4) q
-order by cint, ctinyint, cstring1
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@small_alltypesorc1a
POSTHOOK: Input: default@small_alltypesorc2a
@@ -86,6 +138,34 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_a
+#### A masked pattern was here ####
+-51 NULL -31312632 1086455747 -51.0 NULL NULL Bc7xt12568c451o64LF5 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -337975743 608681041 -51.0 NULL NULL Ih2r28o6 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -413196097 -306198070 -51.0 NULL NULL F53QcSDPpxYF1Ub 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL -591488718 803603078 -51.0 NULL NULL X616UtmmA3FHan 1969-12-31 16:00:08.451 NULL NULL true
+-51 NULL -738306196 -460430946 -51.0 NULL NULL dBOqv 1969-12-31 16:00:08.451 NULL NULL false
+-51 NULL NULL -1731061911 -51.0 NULL Pw53BBJ yL443x2437PO5Hv1U3lCjq2D 1969-12-31 16:00:08.451 NULL true false
+-51 NULL NULL -1846191223 -51.0 NULL Ul085f84S33Xd32u x1JC58g0Ukp 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1874052220 -51.0 NULL c61B47I604gymFJ sjWQS78 1969-12-31 16:00:08.451 NULL false false
+-51 NULL NULL -1927203921 -51.0 NULL 45ja5suO 42S0I0 1969-12-31 16:00:08.451 NULL true true
+-51 NULL NULL -1970551565 -51.0 NULL r2uhJH3 loXMWyrHjVeK 1969-12-31 16:00:08.451 NULL false false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -120,11 +200,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -134,7 +214,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 5024 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -144,7 +224,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5526 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -160,31 +240,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -232,7 +312,7 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-15
+20
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -267,11 +347,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -281,7 +361,7 @@ STAGE PLANS:
outputColumnNames: _col0
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 5024 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -291,7 +371,7 @@ STAGE PLANS:
1 _col0 (type: string)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5526 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -307,31 +387,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -379,7 +459,7 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21
+28
PREHOOK: query: explain
select count(*) from (select c.cstring1
from small_alltypesorc_a c
@@ -414,11 +494,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cstring2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -428,7 +508,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col2
input vertices:
1 Map 3
- Statistics: Num rows: 16 Data size: 4070 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 5024 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Map Join Operator
condition map:
@@ -438,7 +518,7 @@ STAGE PLANS:
1 _col1 (type: string), _col0 (type: int)
input vertices:
1 Map 4
- Statistics: Num rows: 17 Data size: 4477 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 24 Data size: 5526 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
Group By Operator
aggregations: count()
@@ -454,31 +534,31 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cstring2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: bigint)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: bigint)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map 4
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cstring1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
- Statistics: Num rows: 15 Data size: 3700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 4568 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
Reduce Operator Tree:
@@ -526,4 +606,4 @@ left outer join small_alltypesorc_a hd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_a
#### A masked pattern was here ####
-21
+28
[30/61] [abbrv] hive git commit: HIVE-10994 : Hive.moveFile should
not fail on a no-op move (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Posted by xu...@apache.org.
HIVE-10994 : Hive.moveFile should not fail on a no-op move (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d8059d8c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d8059d8c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d8059d8c
Branch: refs/heads/spark
Commit: d8059d8c05442baa1ca3aabf62ac1d05eaccbf65
Parents: b86728d
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jun 15 13:53:55 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jun 15 13:53:55 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d8059d8c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 0e990f6..1428c0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -2616,6 +2616,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
} else {
if (destIsSubDir) {
FileStatus[] srcs = fs.listStatus(srcf, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ if (srcs.length == 0) {
+ success = true; // Nothing to move.
+ }
for (FileStatus status : srcs) {
success = FileUtils.copy(srcf.getFileSystem(conf), status.getPath(), destf.getFileSystem(conf), destf,
true, // delete source
[33/61] [abbrv] hive git commit: HIVE-10704: Errors in Tez
HashTableLoader when estimated table size is 0 (Jason Dere,
reviewed by Alex Pivovarov)
Posted by xu...@apache.org.
HIVE-10704: Errors in Tez HashTableLoader when estimated table size is 0 (Jason Dere, reviewed by Alex Pivovarov)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44279356
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44279356
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44279356
Branch: refs/heads/spark
Commit: 44279356874cba3b42ca2ae3353f07bceb27dbf8
Parents: 3934c18
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Jun 15 15:27:41 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Jun 15 15:31:10 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/HashTableLoader.java | 64 ++++++++++++++++----
1 file changed, 53 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/44279356/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 7e67d40..c688092 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -97,29 +98,33 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
HybridHashTableConf nwayConf = null;
long totalSize = 0;
int biggest = 0; // position of the biggest small table
+ Map<Integer, Long> tableMemorySizes = null;
if (useHybridGraceHashJoin && mapJoinTables.length > 2) {
// Create a Conf for n-way HybridHashTableContainers
nwayConf = new HybridHashTableConf();
// Find the biggest small table; also calculate total data size of all small tables
- long maxSize = 0; // the size of the biggest small table
+ long maxSize = Long.MIN_VALUE; // the size of the biggest small table
for (int pos = 0; pos < mapJoinTables.length; pos++) {
if (pos == desc.getPosBigTable()) {
continue;
}
- totalSize += desc.getParentDataSizes().get(pos);
- biggest = desc.getParentDataSizes().get(pos) > maxSize ? pos : biggest;
- maxSize = desc.getParentDataSizes().get(pos) > maxSize ? desc.getParentDataSizes().get(pos)
- : maxSize;
+ long smallTableSize = desc.getParentDataSizes().get(pos);
+ totalSize += smallTableSize;
+ if (maxSize < smallTableSize) {
+ maxSize = smallTableSize;
+ biggest = pos;
+ }
}
+ tableMemorySizes = divideHybridHashTableMemory(mapJoinTables, desc,
+ totalSize, noConditionalTaskThreshold);
// Using biggest small table, calculate number of partitions to create for each small table
- float percentage = (float) maxSize / totalSize;
- long memory = (long) (noConditionalTaskThreshold * percentage);
+ long memory = tableMemorySizes.get(biggest);
int numPartitions = 0;
try {
numPartitions = HybridHashTableContainer.calcNumPartitions(memory,
- desc.getParentDataSizes().get(biggest),
+ maxSize,
HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHYBRIDGRACEHASHJOINMINNUMPARTITIONS),
HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHYBRIDGRACEHASHJOINMINWBSIZE),
nwayConf);
@@ -169,9 +174,7 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
long memory = 0;
if (useHybridGraceHashJoin) {
if (mapJoinTables.length > 2) {
- // Allocate n-way join memory proportionally
- float percentage = (float) desc.getParentDataSizes().get(pos) / totalSize;
- memory = (long) (noConditionalTaskThreshold * percentage);
+ memory = tableMemorySizes.get(pos);
} else { // binary join
memory = noConditionalTaskThreshold;
}
@@ -196,6 +199,45 @@ public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTable
}
}
+ private static Map<Integer, Long> divideHybridHashTableMemory(
+ MapJoinTableContainer[] mapJoinTables, MapJoinDesc desc,
+ long totalSize, long totalHashTableMemory) {
+ int smallTableCount = Math.max(mapJoinTables.length - 1, 1);
+ Map<Integer, Long> tableMemorySizes = new HashMap<Integer, Long>();
+ // If any table has bad size estimate, we need to fall back to sizing each table equally
+ boolean fallbackToEqualProportions = totalSize <= 0;
+
+ if (!fallbackToEqualProportions) {
+ for (Map.Entry<Integer, Long> tableSizeEntry : desc.getParentDataSizes().entrySet()) {
+ if (tableSizeEntry.getKey() == desc.getPosBigTable()) {
+ continue;
+ }
+
+ long tableSize = tableSizeEntry.getValue();
+ if (tableSize <= 0) {
+ fallbackToEqualProportions = true;
+ break;
+ }
+ float percentage = (float) tableSize / totalSize;
+ long tableMemory = (long) (totalHashTableMemory * percentage);
+ tableMemorySizes.put(tableSizeEntry.getKey(), tableMemory);
+ }
+ }
+
+ if (fallbackToEqualProportions) {
+ // Just give each table the same amount of memory.
+ long equalPortion = totalHashTableMemory / smallTableCount;
+ for (Integer pos : desc.getParentDataSizes().keySet()) {
+ if (pos == desc.getPosBigTable()) {
+ break;
+ }
+ tableMemorySizes.put(pos, equalPortion);
+ }
+ }
+
+ return tableMemorySizes;
+ }
+
private String describeOi(String desc, ObjectInspector keyOi) {
for (StructField field : ((StructObjectInspector)keyOi).getAllStructFieldRefs()) {
ObjectInspector oi = field.getFieldObjectInspector();
[15/61] [abbrv] hive git commit: HIVE-10943 Beeline-cli: Enable precommit for beelie-cli branch(Ferdinand Xu, reviewed by Xuefu Zhang and Sergio Peña)
Posted by xu...@apache.org.
HIVE-10943 Beeline-cli: Enable precommit for beelie-cli branch(Ferdinand Xu, reviewed by Xuefu Zhang and Sergio Peña)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/612603f3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/612603f3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/612603f3
Branch: refs/heads/spark
Commit: 612603f383d8e6a3a35189dcb7af189839ec4076
Parents: d0a9e78
Author: Ferdinand Xu <ch...@intel.com>
Authored: Wed Jun 10 13:55:22 2015 -0400
Committer: Ferdinand Xu <ch...@intel.com>
Committed: Wed Jun 10 13:55:36 2015 -0400
----------------------------------------------------------------------
dev-support/jenkins-submit-build.sh | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/612603f3/dev-support/jenkins-submit-build.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-submit-build.sh b/dev-support/jenkins-submit-build.sh
index 6b2e8e8..a145e00 100644
--- a/dev-support/jenkins-submit-build.sh
+++ b/dev-support/jenkins-submit-build.sh
@@ -44,6 +44,10 @@ case "$BUILD_PROFILE" in
test -n "$PARQUET_URL" || fail "PARQUET_URL must be specified"
url="$PARQUET_URL&ISSUE_NUM=$ISSUE_NUM"
;;
+ beeline-cli-mr2)
+ test -n "$BEELINE_CLI_URL" || fail "BEELINE_CLI_URL must be specified"
+ url="$BEELINE_CLI_URL&ISSUE_NUM=$ISSUE_NUM"
+ ;;
*)
echo "Unknown profile '$BUILD_PROFILE'"
exit 1
[11/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join27.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join27.q.out b/ql/src/test/results/clientpositive/spark/auto_join27.q.out
index 1722837..43313e0 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join27.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join27.q.out
@@ -41,7 +41,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 200) (type: boolean)
+ predicate: (UDFToDouble(key) < 200.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
@@ -58,25 +58,29 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 200) (type: boolean)
+ predicate: (UDFToDouble(key) < 200.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: key (type: string), value (type: string)
- mode: hash
+ Select Operator
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Map 6
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 200) (type: boolean)
+ predicate: (UDFToDouble(key) < 200.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join4.q.out b/ql/src/test/results/clientpositive/spark/auto_join4.q.out
index 3366f75..7cf582c 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join4.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join4.q.out
@@ -50,15 +50,15 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: string)
@@ -76,7 +76,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -154,8 +154,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join5.q.out b/ql/src/test/results/clientpositive/spark/auto_join5.q.out
index b6d8798..08e9ae6 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join5.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join5.q.out
@@ -47,18 +47,18 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 2
Map Operator Tree:
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and (UDFToDouble(key) > 15.0)) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: string)
@@ -70,13 +70,13 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 1
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -84,16 +84,16 @@ STAGE PLANS:
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
- Right Outer Join0 to 1
+ Left Outer Join0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
input vertices:
- 0 Map 1
+ 1 Map 2
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col3 (type: string), UDFToInteger(_col0) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -154,8 +154,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join6.q.out b/ql/src/test/results/clientpositive/spark/auto_join6.q.out
index 2ca2674..5b7e904 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join6.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join6.q.out
@@ -54,7 +54,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -69,10 +69,10 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 15) and (key < 25)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -154,8 +154,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join8.q.out b/ql/src/test/results/clientpositive/spark/auto_join8.q.out
index e0df9a2..e77817a 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join8.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join8.q.out
@@ -50,15 +50,15 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 15) and (key < 25)) and key is not null) (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: string)
@@ -76,7 +76,7 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 10) and (key < 20)) and key is not null) (type: boolean)
+ predicate: (((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -96,7 +96,7 @@ STAGE PLANS:
predicate: _col2 is null (type: boolean)
Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), null (type: int), _col3 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(null) (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 15 Data size: 163 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -157,8 +157,8 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join9.q.out b/ql/src/test/results/clientpositive/spark/auto_join9.q.out
index 6daf348..568891b 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join9.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join9.q.out
@@ -33,10 +33,14 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
@@ -52,28 +56,32 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col8
- input vertices:
- 1 Map 2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col8 (type: string)
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col4 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join_filters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join_filters.q.out b/ql/src/test/results/clientpositive/spark/auto_join_filters.q.out
index 8934433..b6b89c1 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join_filters.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join_filters.q.out
@@ -44,7 +44,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
3080335
-Warning: Shuffle Join JOIN[4][tables = [a, b]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
@@ -330,7 +330,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
3078400
-Warning: Shuffle Join JOIN[10][tables = [a, b]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out b/ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out
index 1f37c75..4b809ce 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join_nulls.q.out
@@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE my
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@myinput1
-Warning: Map Join MAPJOIN[11][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
@@ -24,7 +24,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
13630578
-Warning: Map Join MAPJOIN[11][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
@@ -34,7 +34,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
13630578
-Warning: Map Join MAPJOIN[11][bigTable=b] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
index c3d5225..72f60d0 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out
@@ -24,33 +24,41 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 4
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/cbo_subq_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/cbo_subq_in.q.out b/ql/src/test/results/clientpositive/spark/cbo_subq_in.q.out
index c1f3de7..f6bfad2 100644
--- a/ql/src/test/results/clientpositive/spark/cbo_subq_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/cbo_subq_in.q.out
@@ -69,6 +69,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
PREHOOK: type: QUERY
PREHOOK: Input: default@lineitem
#### A masked pattern was here ####
@@ -77,6 +78,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
POSTHOOK: type: QUERY
POSTHOOK: Input: default@lineitem
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
index 602de6e..665bfce 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
@@ -23,39 +23,45 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 100) (type: boolean)
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: '100' (type: string)
- sort order: +
- Map-reduce partition columns: '100' (type: string)
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '100' (type: string)
+ sort order: +
+ Map-reduce partition columns: '100' (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key = 100) (type: boolean)
+ predicate: (UDFToDouble(key) = 100.0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: '100' (type: string)
- sort order: +
- Map-reduce partition columns: '100' (type: string)
+ Select Operator
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: '100' (type: string)
+ sort order: +
+ Map-reduce partition columns: '100' (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: '100' (type: string), 101.0 (type: double), _col6 (type: string)
+ expressions: '100' (type: string), 101.0 (type: double), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -115,53 +121,57 @@ STAGE PLANS:
alias: li
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((l_orderkey is not null and l_linenumber is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: l_orderkey (type: int), l_linenumber (type: int)
- sort order: ++
- Map-reduce partition columns: l_orderkey (type: int), l_linenumber (type: int)
- Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
- value expressions: l_partkey (type: int), l_suppkey (type: int)
+ predicate: ((l_linenumber = 1) and l_orderkey is not null) (type: boolean)
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col3 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int)
Map 3
Map Operator Tree:
TableScan
- alias: lineitem
+ alias: li
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) and (l_linenumber = 1)) (type: boolean)
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((l_shipmode = 'AIR') and l_orderkey is not null) and l_linenumber is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: l_orderkey (type: int), 1 (type: int)
+ expressions: l_orderkey (type: int), l_linenumber (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 6 Data size: 719 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 1559 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Left Semi Join 0 to 1
keys:
- 0 l_orderkey (type: int), l_linenumber (type: int)
+ 0 _col0 (type: int), _col3 (type: int)
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col1, _col2
- Statistics: Num rows: 13 Data size: 1582 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: int), _col2 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1582 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 13 Data size: 1582 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 3298 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/count.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out
index cb9eda5..b2e9ffb 100644
--- a/ql/src/test/results/clientpositive/spark/count.q.out
+++ b/ql/src/test/results/clientpositive/spark/count.q.out
@@ -53,11 +53,11 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: a (type: int), b (type: int), c (type: int), d (type: int)
- outputColumnNames: a, b, c, d
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT b), count(DISTINCT c), sum(d)
- keys: a (type: int), b (type: int), c (type: int)
+ aggregations: count(DISTINCT _col1), count(DISTINCT _col2), sum(_col3)
+ keys: _col0 (type: int), _col1 (type: int), _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
@@ -188,14 +188,14 @@ STAGE PLANS:
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: a (type: int), b (type: int), c (type: int), d (type: int)
- outputColumnNames: a, b, c, d
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: a (type: int), b (type: int), c (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
sort order: +++
- Map-reduce partition columns: a (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 4 Data size: 78 Basic stats: COMPLETE Column stats: NONE
- value expressions: d (type: int)
+ value expressions: _col3 (type: int)
Reducer 2
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/cross_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/cross_join.q.out b/ql/src/test/results/clientpositive/spark/cross_join.q.out
index bb25e21..2fb6b21 100644
--- a/ql/src/test/results/clientpositive/spark/cross_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/cross_join.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join JOIN[4][tables = [src, src2]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: -- current
explain select src.key from src join src src2
PREHOOK: type: QUERY
@@ -21,18 +21,24 @@ STAGE PLANS:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -57,7 +63,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[4][tables = [src, src2]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: -- ansi cross join
explain select src.key from src cross join src src2
PREHOOK: type: QUERY
@@ -80,18 +86,24 @@ STAGE PLANS:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -141,32 +153,40 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
index ff58910..c228f0b 100644
--- a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
@@ -28,7 +28,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@B
-Warning: Shuffle Join JOIN[4][tables = [a, b]] in Work 'Reducer 2' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Work 'Reducer 2' is a cross product
PREHOOK: query: explain select * from A join B
PREHOOK: type: QUERY
POSTHOOK: query: explain select * from A join B
@@ -49,19 +49,27 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string), value (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
alias: b
Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string), value (type: string)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -70,19 +78,15 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0, _col1, _col5, _col6
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ File Output Operator
+ compressed: false
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out b/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out
index 6c4e659..ab062ad 100644
--- a/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out
+++ b/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out
@@ -28,7 +28,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@B
-Warning: Map Join MAPJOIN[7][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain select * from A join B
PREHOOK: type: QUERY
POSTHOOK: query: explain select * from A join B
@@ -48,10 +48,14 @@ STAGE PLANS:
TableScan
alias: b
Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0
- 1
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 10 Data size: 96 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0
+ 1
Local Work:
Map Reduce Local Work
@@ -64,19 +68,19 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col0, _col1, _col5, _col6
- input vertices:
- 1 Map 2
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
outputColumnNames: _col0, _col1, _col2, _col3
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby1.q.out b/ql/src/test/results/clientpositive/spark/groupby1.q.out
index f661a62..8f60691 100644
--- a/ql/src/test/results/clientpositive/spark/groupby1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby1.q.out
@@ -35,15 +35,15 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: key (type: string), substr(value, 5) (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: key (type: string)
+ key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: rand() (type: double)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: substr(value, 5) (type: string)
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby4.q.out b/ql/src/test/results/clientpositive/spark/groupby4.q.out
index 87d2968..b764466 100644
--- a/ql/src/test/results/clientpositive/spark/groupby4.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby4.q.out
@@ -37,11 +37,11 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ expressions: substr(key, 1, 1) (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: substr(key, 1, 1) (type: string)
+ key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: rand() (type: double)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out b/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out
index 9fe3b72..65bdf2c 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out
@@ -20,13 +20,13 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@DEST2
PREHOOK: query: EXPLAIN
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
@@ -41,8 +41,8 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 6 (GROUP, 2)
Reducer 4 <- Map 7 (GROUP, 2)
- Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 5 <- Reducer 4 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
+ Reducer 5 <- Reducer 4 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 6
@@ -95,19 +95,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: array<string>), _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: array<string>)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: array<string>), VALUE._col1 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: array<string>), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -127,19 +125,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: map<string,string>), _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: map<string,string>)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 5
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: map<string,string>), VALUE._col1 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: map<string,string>), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -179,15 +175,15 @@ STAGE PLANS:
Stats-Aggr Operator
PREHOOK: query: FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@dest1
PREHOOK: Output: default@dest2
POSTHOOK: query: FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
@@ -204,16 +200,16 @@ POSTHOOK: query: SELECT DEST1.* FROM DEST1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
-["118"] 2
-["180"] 1
-["201"] 1
-["202"] 1
-["238"] 2
-["273"] 3
-["282"] 2
-["419"] 1
-["432"] 1
-["467"] 1
+["0"] 3
+["10"] 1
+["100"] 2
+["103"] 2
+["104"] 2
+["105"] 1
+["11"] 1
+["111"] 1
+["113"] 2
+["114"] 1
PREHOOK: query: SELECT DEST2.* FROM DEST2
PREHOOK: type: QUERY
PREHOOK: Input: default@dest2
@@ -223,12 +219,12 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest2
#### A masked pattern was here ####
{"0":"val_0"} 3
-{"138":"val_138"} 4
-{"170":"val_170"} 1
-{"19":"val_19"} 1
-{"222":"val_222"} 1
-{"223":"val_223"} 2
-{"226":"val_226"} 1
-{"489":"val_489"} 4
-{"8":"val_8"} 1
-{"80":"val_80"} 1
+{"10":"val_10"} 1
+{"100":"val_100"} 2
+{"103":"val_103"} 2
+{"104":"val_104"} 2
+{"105":"val_105"} 1
+{"11":"val_11"} 1
+{"111":"val_111"} 1
+{"113":"val_113"} 2
+{"114":"val_114"} 1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
index 42513a4..21c88b1 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out
@@ -42,11 +42,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4 Data size: 120 Basic stats: COMPLETE Column stats: NONE
@@ -133,11 +133,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4 Data size: 120 Basic stats: COMPLETE Column stats: NONE
@@ -223,11 +223,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT val)
- keys: key (type: string), '0' (type: string), val (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), '0' (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
@@ -302,11 +302,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 4 Data size: 120 Basic stats: COMPLETE Column stats: NONE
@@ -407,11 +407,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT val)
- keys: key (type: string), '0' (type: string), val (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), '0' (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_position.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
index 16437d5..50d707d 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out
@@ -436,20 +436,24 @@ STAGE PLANS:
alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <= 20) (type: boolean)
+ predicate: (UDFToDouble(key) <= 20.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -548,7 +552,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (GROUP, 2)
Reducer 6 <- Map 5 (GROUP, 2)
Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2)
Reducer 4 <- Reducer 3 (SORT, 1)
@@ -560,55 +564,57 @@ STAGE PLANS:
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 10) and (key < 20)) and key is not null) (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(DISTINCT substr(value, 5))
- keys: key (type: string), value (type: string), substr(value, 5) (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
- sort order: +++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and (UDFToDouble(key) > 15.0)) and (UDFToDouble(key) < 25.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Map 5
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((key > 15) and (key < 25)) and key is not null) (type: boolean)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: key (type: string), value (type: string)
- mode: hash
+ predicate: (((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col2:0._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
@@ -617,25 +623,21 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col3, _col4
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
- sort order: --++
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ sort order: --++
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -646,12 +648,12 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Stage: Stage-0
[10/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
index 6f4854c..92b6d19 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out
@@ -42,11 +42,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 90 Basic stats: COMPLETE Column stats: NONE
@@ -127,11 +127,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT val)
- keys: key (type: string), '0' (type: string), val (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), '0' (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
@@ -206,11 +206,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 90 Basic stats: COMPLETE Column stats: NONE
@@ -305,11 +305,11 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 30 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT val)
- keys: key (type: string), '0' (type: string), val (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), '0' (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 60 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
index 65a8374..6c0f4a5 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
@@ -97,11 +97,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -308,11 +308,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -980,11 +980,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: 1 (type: int), key (type: string)
+ keys: 1 (type: int), _col1 (type: string)
mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -1195,11 +1195,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -1421,12 +1421,12 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ expressions: key (type: string), (UDFToDouble(key) + 1.0) (type: double)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), (key + 1) (type: double)
+ keys: _col0 (type: string), _col1 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -1685,28 +1685,32 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col1)
- keys: (_col0 + _col0) (type: double)
- mode: hash
+ Select Operator
+ expressions: (UDFToDouble(_col0) + UDFToDouble(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Map-reduce partition columns: _col0 (type: double)
+ Group By Operator
+ aggregations: sum(_col1)
+ keys: _col0 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: bigint)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1958,11 +1962,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2062,11 +2066,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2330,11 +2334,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2437,12 +2441,12 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ expressions: (UDFToDouble(key) + UDFToDouble(key)) (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: (key + key) (type: double)
+ keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -3386,12 +3390,12 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
bucketGroup: true
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -3620,11 +3624,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -3850,11 +3854,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string), 2 (type: int)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
index 9f30e15..8248a70 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
@@ -97,11 +97,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -309,11 +309,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -998,11 +998,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: 1 (type: int), key (type: string)
+ keys: 1 (type: int), _col1 (type: string)
mode: final
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -1214,11 +1214,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -1458,12 +1458,12 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ expressions: key (type: string), (UDFToDouble(key) + 1.0) (type: double)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), (key + 1) (type: double)
+ keys: _col0 (type: string), _col1 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -1740,28 +1740,32 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col1)
- keys: (_col0 + _col0) (type: double)
- mode: hash
+ Select Operator
+ expressions: (UDFToDouble(_col0) + UDFToDouble(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Map-reduce partition columns: rand() (type: double)
+ Group By Operator
+ aggregations: sum(_col1)
+ keys: _col0 (type: double)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col1 (type: bigint)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -2030,11 +2034,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2134,11 +2138,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2403,11 +2407,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -2510,12 +2514,12 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Select Operator
- expressions: key (type: string)
- outputColumnNames: key
+ expressions: (UDFToDouble(key) + UDFToDouble(key)) (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: (key + key) (type: double)
+ keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -3495,12 +3499,12 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
bucketGroup: true
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
@@ -3746,11 +3750,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
@@ -3976,11 +3980,11 @@ STAGE PLANS:
GatherStats: false
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), 1 (type: int), val (type: string), 2 (type: int)
+ keys: _col0 (type: string), 1 (type: int), _col2 (type: string), 2 (type: int)
mode: final
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/having.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out
index da6186e..245b27a 100644
--- a/ql/src/test/results/clientpositive/spark/having.q.out
+++ b/ql/src/test/results/clientpositive/spark/having.q.out
@@ -104,20 +104,24 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <> 302) (type: boolean)
+ predicate: (UDFToDouble(key) <> 302.0) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: max(value)
- keys: key (type: string)
- mode: hash
+ Select Operator
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: max(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -478,11 +482,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(value)
- keys: key (type: string)
+ aggregations: max(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -749,20 +753,24 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key > 300) (type: boolean)
+ predicate: (UDFToDouble(key) > 300.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: max(value)
- keys: key (type: string)
- mode: hash
+ Select Operator
+ expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: max(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -947,11 +955,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: max(value)
- keys: key (type: string)
+ aggregations: max(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -1215,11 +1223,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(value)
- keys: key (type: string)
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/index_auto_self_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/index_auto_self_join.q.out b/ql/src/test/results/clientpositive/spark/index_auto_self_join.q.out
index 57942af..c965455 100644
--- a/ql/src/test/results/clientpositive/spark/index_auto_self_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/index_auto_self_join.q.out
@@ -25,45 +25,53 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((value is not null and (key > 80)) and (key < 100)) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and value is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((value is not null and (key > 70)) and (key < 90)) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ predicate: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and value is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 value (type: string)
- 1 value (type: string)
- outputColumnNames: _col0, _col5
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col5 (type: string)
+ expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -133,49 +141,57 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: a
- filterExpr: ((value is not null and (key > 80)) and (key < 100)) (type: boolean)
+ filterExpr: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and value is not null) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((value is not null and (key > 80)) and (key < 100)) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and value is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: b
- filterExpr: ((value is not null and (key > 70)) and (key < 90)) (type: boolean)
+ alias: a
+ filterExpr: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and value is not null) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((value is not null and (key > 70)) and (key < 90)) (type: boolean)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string)
+ predicate: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and value is not null) (type: boolean)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 value (type: string)
- 1 value (type: string)
- outputColumnNames: _col0, _col5
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col5 (type: string)
+ expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 29 Data size: 314 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 326 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
index 4da17fb..94adb3d 100644
--- a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
+++ b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
@@ -79,11 +79,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: final
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -164,6 +164,7 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
+WARNING: Comparing a bigint and a string may result in a loss of precision.
PREHOOK: query: -- Test map group by doesn't affect inference, should be bucketed and sorted by value
EXPLAIN INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1')
SELECT a.key, a.value FROM (
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out
index 0f352d7..d947eb5 100644
--- a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out
+++ b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out
@@ -27,7 +27,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@test_table@part=1
POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)a.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1')
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@test_table
@@ -82,7 +82,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@test_table@part=1
POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)a.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1')
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@test_table
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/innerjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/innerjoin.q.out b/ql/src/test/results/clientpositive/spark/innerjoin.q.out
index 3f6f9ab..762ddad 100644
--- a/ql/src/test/results/clientpositive/spark/innerjoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/innerjoin.q.out
@@ -38,37 +38,45 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -104,7 +112,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest_j1.* FROM dest_j1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
@@ -1214,12 +1222,16 @@ STAGE PLANS:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -1227,7 +1239,7 @@ STAGE PLANS:
Left Outer Join0 to 1
keys:
0 _col0 (type: string)
- 1 key (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join1.q.out b/ql/src/test/results/clientpositive/spark/join1.q.out
index 145bbe4..4714f11 100644
--- a/ql/src/test/results/clientpositive/spark/join1.q.out
+++ b/ql/src/test/results/clientpositive/spark/join1.q.out
@@ -38,37 +38,45 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -104,7 +112,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest_j1.* FROM dest_j1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join10.q.out b/ql/src/test/results/clientpositive/spark/join10.q.out
index e7dd570..8d18683 100644
--- a/ql/src/test/results/clientpositive/spark/join10.q.out
+++ b/ql/src/test/results/clientpositive/spark/join10.q.out
@@ -36,14 +36,15 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
@@ -53,15 +54,14 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -70,19 +70,15 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col2, _col3
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: string), _col3 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join11.q.out b/ql/src/test/results/clientpositive/spark/join11.q.out
index 2b5eb3a..7d59d0c 100644
--- a/ql/src/test/results/clientpositive/spark/join11.q.out
+++ b/ql/src/test/results/clientpositive/spark/join11.q.out
@@ -35,35 +35,35 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -72,10 +72,10 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col3
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col3 (type: string)
+ expressions: _col2 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join14.q.out b/ql/src/test/results/clientpositive/spark/join14.q.out
index 336787b..e942087 100644
--- a/ql/src/test/results/clientpositive/spark/join14.q.out
+++ b/ql/src/test/results/clientpositive/spark/join14.q.out
@@ -35,42 +35,50 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (key > 100) (type: boolean)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Map 3
- Map Operator Tree:
- TableScan
alias: srcpart
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 100) and key is not null) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and key is not null) (type: boolean)
Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (UDFToDouble(key) > 100.0) (type: boolean)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col3
Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col3) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join16.q.out b/ql/src/test/results/clientpositive/spark/join16.q.out
index 000ea08..aecd953 100644
--- a/ql/src/test/results/clientpositive/spark/join16.q.out
+++ b/ql/src/test/results/clientpositive/spark/join16.q.out
@@ -19,30 +19,34 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean)
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) > 20.0)) and (UDFToDouble(value) < 200.0)) and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
- alias: tab
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((UDFToDouble(value) < 200.0) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) > 20.0)) and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -50,16 +54,16 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: string), _col1 (type: string)
- 1 key (type: string), value (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join17.q.out b/ql/src/test/results/clientpositive/spark/join17.q.out
index 4aa2a05..4aefeb3 100644
--- a/ql/src/test/results/clientpositive/spark/join17.q.out
+++ b/ql/src/test/results/clientpositive/spark/join17.q.out
@@ -78,14 +78,18 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -138,21 +142,25 @@ STAGE PLANS:
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -201,7 +209,7 @@ STAGE PLANS:
name: default.src
name: default.src
Truncated Path -> Alias:
- /src [src2]
+ /src [src1]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
@@ -209,12 +217,12 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col5) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -281,9 +289,9 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.key1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.value1 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/join18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join18.q.out b/ql/src/test/results/clientpositive/spark/join18.q.out
index 3d266ad..ed17bab 100644
--- a/ql/src/test/results/clientpositive/spark/join18.q.out
+++ b/ql/src/test/results/clientpositive/spark/join18.q.out
@@ -48,11 +48,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(value)
- keys: key (type: string)
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -69,11 +69,11 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT value)
- keys: key (type: string), value (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
[14/61] [abbrv] hive git commit: HIVE-10963 : Hive throws NPE rather
than meaningful error message when window is missing (Aihua Xu via Ashutosh
Chauhan)
Posted by xu...@apache.org.
HIVE-10963 : Hive throws NPE rather than meaningful error message when window is missing (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f2600e9d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f2600e9d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f2600e9d
Branch: refs/heads/spark
Commit: f2600e9da833f409afc33be9f92db4dbe9ad2c9b
Parents: 19d7074
Author: Aihua Xu <ai...@gmail.com>
Authored: Tue Jun 9 07:41:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jun 10 00:13:46 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 6 +--
.../hadoop/hive/ql/parse/PTFInvocationSpec.java | 8 ++++
.../hadoop/hive/ql/parse/WindowingSpec.java | 46 ++++++++++----------
.../clientnegative/ptf_negative_NoWindowDefn.q | 9 ++++
.../ptf_negative_NoWindowDefn.q.out | 1 +
5 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f2600e9d/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 48e3cc7..bff9772 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -2378,9 +2378,9 @@ public class CalcitePlanner extends SemanticAnalyzer {
WindowSpec wndSpec = ((WindowFunctionSpec) wExpSpec).getWindowSpec();
List<RexNode> partitionKeys = getPartitionKeys(wndSpec.getPartition(), converter, inputRR);
List<RexFieldCollation> orderKeys = getOrderKeys(wndSpec.getOrder(), converter, inputRR);
- RexWindowBound upperBound = getBound(wndSpec.windowFrame.start, converter);
- RexWindowBound lowerBound = getBound(wndSpec.windowFrame.end, converter);
- boolean isRows = ((wndSpec.windowFrame.start instanceof RangeBoundarySpec) || (wndSpec.windowFrame.end instanceof RangeBoundarySpec)) ? true
+ RexWindowBound upperBound = getBound(wndSpec.getWindowFrame().start, converter);
+ RexWindowBound lowerBound = getBound(wndSpec.getWindowFrame().end, converter);
+ boolean isRows = ((wndSpec.getWindowFrame().start instanceof RangeBoundarySpec) || (wndSpec.getWindowFrame().end instanceof RangeBoundarySpec)) ? true
: false;
w = cluster.getRexBuilder().makeOver(calciteAggFnRetType, calciteAggFn, calciteAggFnArgs,
http://git-wip-us.apache.org/repos/asf/hive/blob/f2600e9d/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
index 06d3f4b..29b8510 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java
@@ -202,6 +202,7 @@ public class PTFInvocationSpec {
public static class PartitioningSpec {
PartitionSpec partSpec;
OrderSpec orderSpec;
+
public PartitionSpec getPartSpec() {
return partSpec;
}
@@ -250,6 +251,13 @@ public class PTFInvocationSpec {
}
return true;
}
+
+ @Override
+ public String toString() {
+ return String.format("PartitioningSpec=[%s%s]",
+ partSpec == null ? "" : partSpec,
+ orderSpec == null ? "" : orderSpec);
+ }
}
/*
http://git-wip-us.apache.org/repos/asf/hive/blob/f2600e9d/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
index 953f3ae..a181f7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
@@ -58,20 +58,21 @@ import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec;
* building RowResolvers.
*/
public class WindowingSpec {
- HashMap<String, WindowExpressionSpec> aliasToWdwExpr;
- HashMap<String, WindowSpec> windowSpecs;
- ArrayList<WindowExpressionSpec> windowExpressions;
+ private HashMap<String, WindowExpressionSpec> aliasToWdwExpr;
+ private HashMap<String, WindowSpec> windowSpecs;
+ private ArrayList<WindowExpressionSpec> windowExpressions;
+
+ public WindowingSpec() {
+ aliasToWdwExpr = new HashMap<String, WindowExpressionSpec>();
+ windowSpecs = new HashMap<String, WindowSpec>();
+ windowExpressions = new ArrayList<WindowExpressionSpec>();
+ }
public void addWindowSpec(String name, WindowSpec wdwSpec) {
- windowSpecs = windowSpecs == null ? new HashMap<String, WindowSpec>() : windowSpecs;
windowSpecs.put(name, wdwSpec);
}
public void addWindowFunction(WindowFunctionSpec wFn) {
- windowExpressions = windowExpressions == null ?
- new ArrayList<WindowExpressionSpec>() : windowExpressions;
- aliasToWdwExpr = aliasToWdwExpr == null ?
- new HashMap<String, WindowExpressionSpec>() : aliasToWdwExpr;
windowExpressions.add(wFn);
aliasToWdwExpr.put(wFn.getAlias(), wFn);
}
@@ -80,26 +81,14 @@ public class WindowingSpec {
return aliasToWdwExpr;
}
- public void setAliasToWdwExpr(HashMap<String, WindowExpressionSpec> aliasToWdwExpr) {
- this.aliasToWdwExpr = aliasToWdwExpr;
- }
-
public HashMap<String, WindowSpec> getWindowSpecs() {
return windowSpecs;
}
- public void setWindowSpecs(HashMap<String, WindowSpec> windowSpecs) {
- this.windowSpecs = windowSpecs;
- }
-
public ArrayList<WindowExpressionSpec> getWindowExpressions() {
return windowExpressions;
}
- public void setWindowExpressions(ArrayList<WindowExpressionSpec> windowExpressions) {
- this.windowExpressions = windowExpressions;
- }
-
public PartitioningSpec getQueryPartitioningSpec() {
/*
* Why no null and class checks?
@@ -171,7 +160,7 @@ public class WindowingSpec {
WindowSpec source = getWindowSpecs().get(sourceId);
if (source == null || source.equals(dest))
{
- throw new SemanticException(String.format("Window Spec %s refers to an unknown source " ,
+ throw new SemanticException(String.format("%s refers to an unknown source" ,
dest));
}
@@ -445,9 +434,10 @@ public class WindowingSpec {
*/
public static class WindowSpec
{
- String sourceId;
- PartitioningSpec partitioning;
- WindowFrameSpec windowFrame;
+ private String sourceId;
+ private PartitioningSpec partitioning;
+ private WindowFrameSpec windowFrame;
+
public String getSourceId() {
return sourceId;
}
@@ -496,6 +486,14 @@ public class WindowingSpec {
setOrder(order);
}
}
+
+ @Override
+ public String toString() {
+ return String.format("Window Spec=[%s%s%s]",
+ sourceId == null ? "" : "Name='" + sourceId + "'",
+ partitioning == null ? "" : partitioning,
+ windowFrame == null ? "" : windowFrame);
+ }
};
/*
http://git-wip-us.apache.org/repos/asf/hive/blob/f2600e9d/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q b/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q
new file mode 100644
index 0000000..8defb3a
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q
@@ -0,0 +1,9 @@
+-- testNoWindowDefn
+select p_mfgr, p_name, p_size,
+sum(p_size) over (w1) as s1,
+sum(p_size) over (w2) as s2
+from part
+distribute by p_mfgr
+sort by p_mfgr
+window w1 as (rows between 2 preceding and 2 following);
+
http://git-wip-us.apache.org/repos/asf/hive/blob/f2600e9d/ql/src/test/results/clientnegative/ptf_negative_NoWindowDefn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ptf_negative_NoWindowDefn.q.out b/ql/src/test/results/clientnegative/ptf_negative_NoWindowDefn.q.out
new file mode 100644
index 0000000..74b97af
--- /dev/null
+++ b/ql/src/test/results/clientnegative/ptf_negative_NoWindowDefn.q.out
@@ -0,0 +1 @@
+FAILED: SemanticException Window Spec=[Name='w2'] refers to an unknown source
[53/61] [abbrv] hive git commit: Updating release notes to reflect
1.2.1
Posted by xu...@apache.org.
Updating release notes to reflect 1.2.1
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/56cffb2d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/56cffb2d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/56cffb2d
Branch: refs/heads/spark
Commit: 56cffb2d1edc97825ee1d093e3d8e46dab6cdf4b
Parents: b98a30b
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Fri Jun 19 01:54:09 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Fri Jun 19 01:54:09 2015 -0700
----------------------------------------------------------------------
NOTICE | 2 +-
RELEASE_NOTES.txt | 132 +++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 133 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/56cffb2d/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index 706f93f..32d89b1 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Hive
-Copyright 2008-2014 The Apache Software Foundation
+Copyright 2008-2015 The Apache Software Foundation
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/hive/blob/56cffb2d/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 6c18cd9..1e92566 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,4 +1,136 @@
+Release Notes - Hive - Version 1.2.1
+
+** Sub-task
+ * [HIVE-10629] - Dropping table in an encrypted zone does not drop warehouse directory
+ * [HIVE-10630] - Renaming tables across encryption zones renames table even though the operation throws error
+ * [HIVE-10658] - Insert with values clause may expose data that should be encrypted
+ * [HIVE-10747] - Enable the cleanup of side effect for the Encryption related qfile test
+ * [HIVE-10910] - Alter table drop partition queries in encrypted zone failing to remove data from HDFS
+
+
+** Bug
+ * [HIVE-4577] - hive CLI can't handle hadoop dfs command with space and quotes.
+ * [HIVE-8931] - Test TestAccumuloCliDriver is not completing
+ * [HIVE-9069] - Simplify filter predicates for CBO
+ * [HIVE-9828] - Semantic analyzer does not capture view parent entity for tables referred in view with union all
+ * [HIVE-9842] - Enable session/operation timeout by default in HiveServer2
+ * [HIVE-10107] - Union All : Vertex missing stats resulting in OOM and in-efficient plans
+ * [HIVE-10244] - Vectorization : TPC-DS Q80 fails with java.lang.ClassCastException when hive.vectorized.execution.reduce.enabled is enabled
+ * [HIVE-10453] - HS2 leaking open file descriptors when using UDFs
+ * [HIVE-10528] - Hiveserver2 in HTTP mode is not applying auth_to_local rules
+ * [HIVE-10563] - MiniTezCliDriver tests ordering issues
+ * [HIVE-10605] - Make hive version number update automatically in webhcat-default.xml during hive tar generation
+ * [HIVE-10606] - Divide by zero error in HybridHashTableContainer
+ * [HIVE-10627] - Queries fail with Failed to breakup Windowing invocations into Groups
+ * [HIVE-10628] - Incorrect result when vectorized native mapjoin is enabled using null safe operators <=>
+ * [HIVE-10659] - Beeline command which contains semi-colon as a non-command terminator will fail
+ * [HIVE-10664] - Unit tests run fail in windows because of illegal escape character in file path
+ * [HIVE-10672] - Analyze command on a table using row format serde JsonSerDe fails with NoClassDefFoundError
+ * [HIVE-10674] - jars should not be checked in to the source control repo
+ * [HIVE-10675] - Provide option to skip Accumulo related Hive tests in itests directory
+ * [HIVE-10677] - hive.exec.parallel=true has problem when it is used for analyze table column stats
+ * [HIVE-10678] - update sql standard authorization configuration whitelist - more optimization flags
+ * [HIVE-10679] - JsonSerde ignores varchar and char size limit specified during table creation
+ * [HIVE-10684] - Fix the unit test failures for HIVE-7553 after HIVE-10674 removed the binary jar files
+ * [HIVE-10685] - Alter table concatenate oparetor will cause duplicate data
+ * [HIVE-10686] - java.lang.IndexOutOfBoundsException for query with rank() over(partition ...)
+ * [HIVE-10688] - constant folding is broken for case-when udf
+ * [HIVE-10689] - HS2 metadata api calls should use HiveAuthorizer interface for authorization
+ * [HIVE-10690] - ArrayIndexOutOfBounds exception in MetaStoreDirectSql.aggrColStatsForPartitions()
+ * [HIVE-10696] - TestAddResource tests are non-portable
+ * [HIVE-10704] - Errors in Tez HashTableLoader when estimated table size is 0
+ * [HIVE-10711] - Tez HashTableLoader attempts to allocate more memory than available when HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD exceeds process max mem
+ * [HIVE-10719] - Hive metastore failure when alter table rename is attempted.
+ * [HIVE-10724] - WebHCat e2e test TestStreaming_5 fails on Windows
+ * [HIVE-10727] - Import throws error message "org.apache.thrift.protocol.TProtocolException: Required field 'filesAdded' is unset!"
+ * [HIVE-10735] - Cached plan race condition - VectorMapJoinCommonOperator has no closeOp()
+ * [HIVE-10736] - HiveServer2 shutdown of cached tez app-masters is not clean
+ * [HIVE-10741] - count distinct rewrite is not firing
+ * [HIVE-10745] - Better null handling by Vectorizer
+ * [HIVE-10746] - Hive 1.2.0+Tez produces 1-byte FileSplits from mapred.TextInputFormat
+ * [HIVE-10753] - hs2 jdbc url - wrong connection string cause error on beeline/jdbc/odbc client, misleading message
+ * [HIVE-10760] - Templeton: HCatalog Get Column for Non-existent column returns Server Error (500) rather than Not Found(404)
+ * [HIVE-10768] - In QTestGenTask.execute() we should not throw an exception right away if we are unable to clean any old files
+ * [HIVE-10776] - Schema on insert for bucketed tables throwing NullPointerException
+ * [HIVE-10789] - union distinct query with NULL constant on both the sides throws "Unsuported vector output type: void" error
+ * [HIVE-10792] - PPD leads to wrong answer when mapper scans the same table with multiple aliases
+ * [HIVE-10801] - 'drop view' fails throwing java.lang.NullPointerException
+ * [HIVE-10805] - OOM in vectorized reduce
+ * [HIVE-10811] - RelFieldTrimmer throws NoSuchElementException in some cases
+ * [HIVE-10813] - Fix current test failures after HIVE-8769
+ * [HIVE-10819] - SearchArgumentImpl for Timestamp is broken by HIVE-10286
+ * [HIVE-10828] - Insert with schema and dynamic partitions NullPointerException
+ * [HIVE-10829] - ATS hook fails for explainTask
+ * [HIVE-10839] - TestHCatLoaderEncryption.* tests fail in windows because of path related issues
+ * [HIVE-10840] - NumberFormatException while running analyze table partition compute statics query
+ * [HIVE-10841] - [WHERE col is not null] does not work sometimes for queries with many JOIN statements
+ * [HIVE-10843] - desc database and show tables commands don't pass db to HiveAuthorizer check
+ * [HIVE-10845] - TezJobMonitor uses killedTaskCount instead of killedTaskAttemptCount
+ * [HIVE-10857] - Accumulo storage handler fail throwing java.lang.IllegalArgumentException: Cannot determine SASL mechanism for token class: class org.apache.accumulo.core.client.security.tokens.PasswordToken
+ * [HIVE-10858] - WebHCat specific resources should be added to HADOOP_CLASSPATH first
+ * [HIVE-10862] - TestHiveAuthorizerShowFilters tests fail when run in sequence
+ * [HIVE-10874] - Fail in TestMinimrCliDriver.testCliDriver_ql_rewrite_gbtoidx_cbo_2.q due to duplicate column name
+ * [HIVE-10875] - Select query with view in subquery adds underlying table as direct input
+ * [HIVE-10877] - TestUtil class name confuses ptest2
+ * [HIVE-10885] - with vectorization enabled join operation involving interval_day_time fails
+ * [HIVE-10887] - TestCliDriver tests ordering issues with Mac and CentOS
+ * [HIVE-10892] - TestHCatClient should not accept external metastore param from -Dhive.metastore.uris
+ * [HIVE-10907] - Hive on Tez: Classcast exception in some cases with SMB joins
+ * [HIVE-10919] - Windows: create table with JsonSerDe failed via beeline unless you add hcatalog core jar to classpath
+ * [HIVE-10922] - In HS2 doAs=false mode, file system related errors in one query causes other failures
+ * [HIVE-10925] - Non-static threadlocals in metastore code can potentially cause memory leak
+ * [HIVE-10929] - In Tez mode,dynamic partitioning query with union all fails at moveTask,Invalid partition key & values
+ * [HIVE-10934] - Restore support for DROP PARTITION PURGE
+ * [HIVE-10941] - Provide option to disable spark tests outside itests
+ * [HIVE-10949] - Disable hive-minikdc tests in Windows
+ * [HIVE-10957] - QueryPlan's start time is incorrect in certain cases
+ * [HIVE-10958] - Centos: TestMiniTezCliDriver.testCliDriver_mergejoin fails
+ * [HIVE-10959] - webhcat launcher job should reconnect to the running child job on task retry
+ * [HIVE-10965] - direct SQL for stats fails in 0-column case
+ * [HIVE-10966] - direct SQL for stats has a cast exception on some databases
+ * [HIVE-10967] - add mapreduce.job.tags to sql std authorization config whitelist
+ * [HIVE-10968] - Windows: analyze json table via beeline failed throwing Class org.apache.hive.hcatalog.data.JsonSerDe not found
+ * [HIVE-10971] - count(*) with count(distinct) gives wrong results when hive.groupby.skewindata=true
+ * [HIVE-10992] - WebHCat should not create delegation tokens when Kerberos is not enabled
+ * [HIVE-10994] - Hive.moveFile should not fail on a no-op move
+ * [HIVE-11001] - HS2 http cookie mode does not honor doAs url parameter
+ * [HIVE-11006] - improve logging wrt ACID module
+ * [HIVE-11010] - Accumulo storage handler queries via HS2 fail
+ * [HIVE-11023] - Disable directSQL if datanucleus.identifierFactory = datanucleus2
+ * [HIVE-11027] - Hive on tez: Bucket map joins fail when hashcode goes negative
+ * [HIVE-11031] - ORC concatenation of old files can fail while merging column statistics
+ * [HIVE-11033] - BloomFilter index is not honored by ORC reader
+ * [HIVE-11035] - PPD: Orc Split elimination fails because filterColumns=[-1]
+ * [HIVE-11040] - Change Derby dependency version to 10.10.2.0
+ * [HIVE-11041] - Update tests for HIVE-9302 after removing binaries
+ * [HIVE-11047] - Update versions of branch-1.2 to 1.2.1
+
+
+** Improvement
+ * [HIVE-8769] - Physical optimizer : Incorrect CE results in a shuffle join instead of a Map join (PK/FK pattern not detected)
+ * [HIVE-10812] - Scaling PK/FK's selectivity for stats annotation
+ * [HIVE-10900] - Fix the indeterministic stats for some hive queries
+
+
+** New Feature
+ * [HIVE-10636] - CASE comparison operator rotation optimization
+ * [HIVE-10716] - Fold case/when udf for expression involving nulls in filter operator.
+
+
+** Task
+ * [HIVE-10327] - Remove ExprNodeNullDesc
+ * [HIVE-11018] - Turn on cbo in more q files
+
+
+** Test
+ * [HIVE-10869] - fold_case.q failing on trunk
+ * [HIVE-10909] - Make TestFilterHooks robust
+ * [HIVE-10939] - Make TestFileDump robust
+ * [HIVE-10955] - CliDriver leaves tables behind at end of test run
+ * [HIVE-11026] - Make vector_outer_join* test more robust
+
+
Release Notes - Hive - Version 1.2.0
[58/61] [abbrv] hive git commit: HIVE-11050:
testCliDriver_vector_outer_join.* failures in Unit tests due to unstable data
creation queries (Matt McCline reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
index 2c9a9cd..7568abf 100644
--- a/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out
@@ -1,43 +1,107 @@
-PREHOOK: query: create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null limit 10
+PREHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1b
-POSTHOOK: query: create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null limit 10
+POSTHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1b
-PREHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null limit 10
+PREHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2b
-POSTHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null limit 10
+POSTHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2b
-PREHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null limit 10
+PREHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3b
-POSTHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null limit 10
+POSTHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3b
-PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null limit 10
+PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4b
-POSTHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null limit 10
+POSTHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4b
+PREHOOK: query: select * from small_alltypesorc1b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1b
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -3097 253665376 NULL -64.0 -3097.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.013 1969-12-31 16:00:06.097 true NULL
+-64 -3586 626923679 NULL -64.0 -3586.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.952 1969-12-31 15:59:51.131 true NULL
+-64 -4018 626923679 NULL -64.0 -4018.0 821UdmGbkEf4j NULL 1969-12-31 15:59:58.959 1969-12-31 16:00:07.803 true NULL
+-64 -4040 528534767 NULL -64.0 -4040.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:47.733 1969-12-31 15:59:46.044 true NULL
+-64 -4803 626923679 NULL -64.0 -4803.0 821UdmGbkEf4j NULL 1969-12-31 16:00:04.662 1969-12-31 16:00:01.609 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+PREHOOK: query: select * from small_alltypesorc2b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2b
+#### A masked pattern was here ####
+-64 -200 NULL -1809444706 -64.0 -200.0 NULL B87YVb3UASqg 1969-12-31 16:00:10.858 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 2118653994 -64.0 -200.0 NULL ONHGSDy1U4Ft431DfQp15 1969-12-31 16:00:03.944 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 927647669 -64.0 -200.0 NULL DhxkBT 1969-12-31 16:00:00.199 1969-12-31 15:59:55.451 NULL false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1090418478 -64.0 -7196.0 NULL 3E06w2 1969-12-31 16:00:00.29 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1805860756 -64.0 -7196.0 NULL 4aOn4s2ATygu0476eD 1969-12-31 16:00:12.339 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3b
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -609074876 -1887561756 NULL NULL EcM71 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:55.061 true false
+NULL NULL -700300206 -1887561756 NULL NULL kdqQE010 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:58.384 false false
+NULL NULL -726473298 1864027286 NULL NULL OFy1a1xf37f75b5N 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:11.799 true true
+NULL NULL -738747840 -1645852809 NULL NULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:11.55 true false
+NULL NULL -838810013 1864027286 NULL NULL N016jPED08o 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:44.252 false true
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc4b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4b
+#### A masked pattern was here ####
PREHOOK: query: create table small_alltypesorc_b stored as orc as select * from
(select * from (select * from small_alltypesorc1b) sq1
union all
@@ -84,6 +148,44 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_b
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -200 NULL -1809444706 -64.0 -200.0 NULL B87YVb3UASqg 1969-12-31 16:00:10.858 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 2118653994 -64.0 -200.0 NULL ONHGSDy1U4Ft431DfQp15 1969-12-31 16:00:03.944 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 927647669 -64.0 -200.0 NULL DhxkBT 1969-12-31 16:00:00.199 1969-12-31 15:59:55.451 NULL false
+-64 -3097 253665376 NULL -64.0 -3097.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.013 1969-12-31 16:00:06.097 true NULL
+-64 -3586 626923679 NULL -64.0 -3586.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.952 1969-12-31 15:59:51.131 true NULL
+-64 -4018 626923679 NULL -64.0 -4018.0 821UdmGbkEf4j NULL 1969-12-31 15:59:58.959 1969-12-31 16:00:07.803 true NULL
+-64 -4040 528534767 NULL -64.0 -4040.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:47.733 1969-12-31 15:59:46.044 true NULL
+-64 -4803 626923679 NULL -64.0 -4803.0 821UdmGbkEf4j NULL 1969-12-31 16:00:04.662 1969-12-31 16:00:01.609 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1090418478 -64.0 -7196.0 NULL 3E06w2 1969-12-31 16:00:00.29 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1805860756 -64.0 -7196.0 NULL 4aOn4s2ATygu0476eD 1969-12-31 16:00:12.339 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -609074876 -1887561756 NULL NULL EcM71 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:55.061 true false
+NULL NULL -700300206 -1887561756 NULL NULL kdqQE010 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:58.384 false false
+NULL NULL -726473298 1864027286 NULL NULL OFy1a1xf37f75b5N 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:11.799 true true
+NULL NULL -738747840 -1645852809 NULL NULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:11.55 true false
+NULL NULL -838810013 1864027286 NULL NULL N016jPED08o 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:44.252 false true
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select *
from small_alltypesorc_b c
@@ -111,11 +213,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -125,11 +227,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
input vertices:
1 Map 2
- Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 7521 Basic stats: COMPLETE Column stats: NONE
HybridGraceHashJoin: true
File Output Operator
compressed: false
- Statistics: Num rows: 33 Data size: 6514 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 7521 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -139,16 +241,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 30 Data size: 5922 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean)
Execution mode: vectorized
@@ -176,416 +278,60 @@ left outer join small_alltypesorc_b cd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
--10 -200 NULL -1818374653 -10.0 -200.0 NULL uFavNs7g58qrfyCH681d 1969-12-31 16:00:03.248 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--21 -200 NULL NULL -21.0 -200.0 NULL NULL 1969-12-31 16:00:09.052 1969-12-31 15:59:55.451 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--30 -200 NULL -815881183 -30.0 -200.0 NULL B0B5kG3OIl6C 1969-12-31 15:59:44.842 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--39 -200 NULL 944477914 -39.0 -200.0 NULL T8brJ213nd7rhW8XdnB1 1969-12-31 16:00:00.958 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--63 -200 NULL 1927856372 -63.0 -200.0 NULL v6mk2b7oX 1969-12-31 16:00:06.852 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-3 -200 NULL -1438142492 3.0 -200.0 NULL T8Uakh8tudd1XRG5yKW8Y42H 1969-12-31 16:00:07.648 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-31 -200 NULL -245476731 31.0 -200.0 NULL 3E3BxP 1969-12-31 15:59:54.739 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-38 -200 NULL -1090414113 38.0 -200.0 NULL 5oQ43l 1969-12-31 16:00:05.478 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-38 -200 NULL 1576772382 38.0 -200.0 NULL X3Ufbt46AUIfHe 1969-12-31 15:59:49.567 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-8 -200 NULL -1236645308 8.0 -200.0 NULL M28wJxOvunO3EImapE7OApQ 1969-12-31 15:59:46.007 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7
<TRUNCATED>
[26/61] [abbrv] hive git commit: HIVE-10968 : Windows: analyze json
table via beeline failed throwing Class
org.apache.hive.hcatalog.data.JsonSerDe not found (Hari Subramaniyan via
Sushanth Sowmyan)
Posted by xu...@apache.org.
HIVE-10968 : Windows: analyze json table via beeline failed throwing Class org.apache.hive.hcatalog.data.JsonSerDe not found (Hari Subramaniyan via Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c3380592
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c3380592
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c3380592
Branch: refs/heads/spark
Commit: c3380592bd0789ef7207f6fc6485342135cfbc1f
Parents: 051bf0b
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Sat Jun 13 19:12:56 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Sat Jun 13 19:12:56 2015 -0700
----------------------------------------------------------------------
bin/ext/hiveserver2.cmd | 51 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 50 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c3380592/bin/ext/hiveserver2.cmd
----------------------------------------------------------------------
diff --git a/bin/ext/hiveserver2.cmd b/bin/ext/hiveserver2.cmd
index 5431de6..26c954c 100644
--- a/bin/ext/hiveserver2.cmd
+++ b/bin/ext/hiveserver2.cmd
@@ -55,6 +55,46 @@ if defined HBASE_HOME (
)
)
+@rem add auxilary jars such as serdes
+if not defined HIVE_AUX_JARS_PATH goto :AddMiscAuxLibDir
+
+setLocal EnableDelayedExpansion
+:auxJarLoop
+ for /f "delims=," %%a in ("!HIVE_AUX_JARS_PATH!") do (
+ set auxjar=%%a
+ if exist %%a (
+ if exist "%%a\nul" (
+ @rem %%a is a dir
+ pushd %%a
+ for /f %%b IN ('dir /b *.jar') do (
+ call :AddToAuxJavaParam %%a\%%b
+ )
+ popd
+ ) else (
+ @rem %%a is a file
+ call :AddToAuxJavaParam %%a
+ )
+ )
+ )
+ :striploop
+ set stripchar=!HIVE_AUX_JARS_PATH:~0,1!
+ set HIVE_AUX_JARS_PATH=!HIVE_AUX_JARS_PATH:~1!
+ if "!HIVE_AUX_JARS_PATH!" EQU "" goto auxJarLoopEnd
+ if "!stripchar!" NEQ "," goto striploop
+ goto auxJarLoop
+
+:auxJarLoopEnd
+
+:AddMiscAuxLibDir
+@rem adding jars from hcatalog\share\hcatalog directory
+if exist %HIVE_HOME%\hcatalog\share\hcatalog (
+ pushd %HIVE_HOME%\hcatalog\share\hcatalog
+ for /f %%a IN ('dir /b *.jar') do (
+ call :AddToAuxJavaParam %HIVE_HOME%\hcatalog\share\hcatalog\%%a
+ )
+ popd
+)
+
if [%1]==[hiveserver2_help] goto :hiveserver2_help
if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice
@@ -76,7 +116,7 @@ goto :EOF
@echo ^<name^>HiveServer2^</name^>
@echo ^<description^>Hadoop HiveServer2 Service^</description^>
@echo ^<executable^>%JAVA_HOME%\bin\java^</executable^>
-@echo ^<arguments^>%JAVA_HEAP_MAX% -XX:MaxPermSize=512m %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH%;%HIVE_HOME%\hcatalog\share\hcatalog\* %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^</arguments^>
+@echo ^<arguments^>%JAVA_HEAP_MAX% -XX:MaxPermSize=512m %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH%;%HIVE_HOME%\hcatalog\share\hcatalog\* %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\*;%HIVE_HOME%\hcatalog\share\hcatalog\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " -hiveconf hive.aux.jars.path=%AUX_JAVA_PARAM% %HIVE_OPTS%^</arguments^>
@echo ^</service^>
goto :EOF
@@ -88,3 +128,12 @@ if not defined HIVE_HBASE_PATH (
)
)
goto :EOF
+
+:AddToAuxJavaParam
+if not defined AUX_JAVA_PARAM (
+ set AUX_JAVA_PARAM=file:///%1
+ ) else (
+ set AUX_JAVA_PARAM=%AUX_JAVA_PARAM%,file:///%1
+ )
+)
+goto :EOF
\ No newline at end of file
[36/61] [abbrv] hive git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/hive
Posted by xu...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/179c9622
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/179c9622
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/179c9622
Branch: refs/heads/spark
Commit: 179c962210f7499f563c6545cbe9b45fd4a496ad
Parents: d6db705 71c3644
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jun 15 17:08:34 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jun 15 17:08:34 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/HashTableLoader.java | 64 ++++-
.../io/orc/OrcFileStripeMergeRecordReader.java | 12 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 21 ++
ql/src/test/queries/clientpositive/tez_smb_1.q | 31 +++
.../results/clientpositive/tez/tez_smb_1.q.out | 278 +++++++++++++++++++
5 files changed, 389 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[47/61] [abbrv] hive git commit: HIVE-11031: ORC concatenation of old
files can fail while merging column statistics (Prasanth Jayachandran
reviewed by Gopal V)
Posted by xu...@apache.org.
HIVE-11031: ORC concatenation of old files can fail while merging column statistics (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f8b0ef8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f8b0ef8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f8b0ef8
Branch: refs/heads/spark
Commit: 3f8b0ef87dfb374038c7170dc8f94c52974872ca
Parents: 8ef6e68
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Thu Jun 18 07:35:54 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Thu Jun 18 07:35:54 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/OrcFileMergeOperator.java | 70 +++---
.../hive/ql/io/orc/ColumnStatisticsImpl.java | 217 ++++++++++++-------
.../hive/ql/io/orc/OrcFileKeyWrapper.java | 27 ++-
.../io/orc/OrcFileStripeMergeRecordReader.java | 22 +-
ql/src/test/queries/clientpositive/orc_merge9.q | 44 ++++
.../results/clientpositive/orc_merge9.q.out | 186 ++++++++++++++++
6 files changed, 434 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index 866f7c0..470c4e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hive.ql.exec;
+import java.io.IOException;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -32,8 +35,6 @@ import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.shims.CombineHiveKey;
-import java.io.IOException;
-
/**
* Fast file merge operator for ORC files.
*/
@@ -72,6 +73,14 @@ public class OrcFileMergeOperator extends
} else {
k = (OrcFileKeyWrapper) key;
}
+
+ // skip incompatible file, files that are missing stripe statistics are set to incompatible
+ if (k.isIncompatFile()) {
+ LOG.warn("Incompatible ORC file merge! Stripe statistics is missing. " + k.getInputPath());
+ incompatFileSet.add(k.getInputPath());
+ return;
+ }
+
filePath = k.getInputPath().toUri().getPath();
fixTmpPath(k.getInputPath().getParent());
@@ -81,9 +90,9 @@ public class OrcFileMergeOperator extends
if (prevPath == null) {
prevPath = k.getInputPath();
reader = OrcFile.createReader(fs, k.getInputPath());
- if (isLogInfoEnabled) {
- LOG.info("ORC merge file input path: " + k.getInputPath());
- }
+ if (isLogInfoEnabled) {
+ LOG.info("ORC merge file input path: " + k.getInputPath());
+ }
}
// store the orc configuration from the first file. All other files should
@@ -102,9 +111,9 @@ public class OrcFileMergeOperator extends
.version(version)
.rowIndexStride(rowIndexStride)
.inspector(reader.getObjectInspector()));
- if (isLogDebugEnabled) {
- LOG.info("ORC merge file output path: " + outPath);
- }
+ if (isLogDebugEnabled) {
+ LOG.info("ORC merge file output path: " + outPath);
+ }
}
if (!checkCompatibility(k)) {
@@ -128,9 +137,10 @@ public class OrcFileMergeOperator extends
v.getStripeStatistics());
if (isLogInfoEnabled) {
- LOG.info("Merged stripe from file " + k.getInputPath() + " [ offset : "
- + v.getStripeInformation().getOffset() + " length: "
- + v.getStripeInformation().getLength() + " ]");
+ LOG.info("Merged stripe from file " + k.getInputPath() + " [ offset : "
+ + v.getStripeInformation().getOffset() + " length: "
+ + v.getStripeInformation().getLength() + " row: "
+ + v.getStripeStatistics().getColStats(0).getNumberOfValues() + " ]");
}
// add user metadata to footer in case of any
@@ -139,9 +149,12 @@ public class OrcFileMergeOperator extends
}
} catch (Throwable e) {
this.exception = true;
- closeOp(true);
+ LOG.error("Closing operator..Exception: " + ExceptionUtils.getStackTrace(e));
throw new HiveException(e);
} finally {
+ if (exception) {
+ closeOp(true);
+ }
if (fdis != null) {
try {
fdis.close();
@@ -157,43 +170,28 @@ public class OrcFileMergeOperator extends
private boolean checkCompatibility(OrcFileKeyWrapper k) {
// check compatibility with subsequent files
if ((k.getTypes().get(0).getSubtypesCount() != columnCount)) {
- if (isLogInfoEnabled) {
- LOG.info("Incompatible ORC file merge! Column counts does not match for "
- + k.getInputPath());
- }
+ LOG.warn("Incompatible ORC file merge! Column counts mismatch for " + k.getInputPath());
return false;
}
if (!k.getCompression().equals(compression)) {
- if (isLogInfoEnabled) {
- LOG.info("Incompatible ORC file merge! Compression codec does not match" +
- " for " + k.getInputPath());
- }
+ LOG.warn("Incompatible ORC file merge! Compression codec mismatch for " + k.getInputPath());
return false;
}
if (k.getCompressBufferSize() != compressBuffSize) {
- if (isLogInfoEnabled) {
- LOG.info("Incompatible ORC file merge! Compression buffer size does not" +
- " match for " + k.getInputPath());
- }
+ LOG.warn("Incompatible ORC file merge! Compression buffer size mismatch for " + k.getInputPath());
return false;
}
if (!k.getVersion().equals(version)) {
- if (isLogInfoEnabled) {
- LOG.info("Incompatible ORC file merge! Version does not match for "
- + k.getInputPath());
- }
+ LOG.warn("Incompatible ORC file merge! Version mismatch for " + k.getInputPath());
return false;
}
if (k.getRowIndexStride() != rowIndexStride) {
- if (isLogInfoEnabled) {
- LOG.info("Incompatible ORC file merge! Row index stride does not match" +
- " for " + k.getInputPath());
- }
+ LOG.warn("Incompatible ORC file merge! Row index stride mismatch for " + k.getInputPath());
return false;
}
@@ -232,7 +230,7 @@ public class OrcFileMergeOperator extends
outWriter.close();
outWriter = null;
- } catch (IOException e) {
+ } catch (Exception e) {
throw new HiveException("Unable to close OrcFileMergeOperator", e);
}
super.closeOp(abort);
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
index ffba3c6..15a3e2c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
@@ -57,9 +57,15 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
+ if (other instanceof BooleanStatisticsImpl) {
+ BooleanStatisticsImpl bkt = (BooleanStatisticsImpl) other;
+ trueCount += bkt.trueCount;
+ } else {
+ if (isStatsExists() && trueCount != 0) {
+ throw new IllegalArgumentException("Incompatible merging of boolean column statistics");
+ }
+ }
super.merge(other);
- BooleanStatisticsImpl bkt = (BooleanStatisticsImpl) other;
- trueCount += bkt.trueCount;
}
@Override
@@ -149,28 +155,35 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl) other;
- if (!hasMinimum) {
- hasMinimum = otherInt.hasMinimum;
- minimum = otherInt.minimum;
- maximum = otherInt.maximum;
- } else if (otherInt.hasMinimum) {
- if (otherInt.minimum < minimum) {
+ if (other instanceof IntegerStatisticsImpl) {
+ IntegerStatisticsImpl otherInt = (IntegerStatisticsImpl) other;
+ if (!hasMinimum) {
+ hasMinimum = otherInt.hasMinimum;
minimum = otherInt.minimum;
- }
- if (otherInt.maximum > maximum) {
maximum = otherInt.maximum;
+ } else if (otherInt.hasMinimum) {
+ if (otherInt.minimum < minimum) {
+ minimum = otherInt.minimum;
+ }
+ if (otherInt.maximum > maximum) {
+ maximum = otherInt.maximum;
+ }
}
- }
- super.merge(other);
- overflow |= otherInt.overflow;
- if (!overflow) {
- boolean wasPositive = sum >= 0;
- sum += otherInt.sum;
- if ((otherInt.sum >= 0) == wasPositive) {
- overflow = (sum >= 0) != wasPositive;
+
+ overflow |= otherInt.overflow;
+ if (!overflow) {
+ boolean wasPositive = sum >= 0;
+ sum += otherInt.sum;
+ if ((otherInt.sum >= 0) == wasPositive) {
+ overflow = (sum >= 0) != wasPositive;
+ }
+ }
+ } else {
+ if (isStatsExists() && hasMinimum) {
+ throw new IllegalArgumentException("Incompatible merging of integer column statistics");
}
}
+ super.merge(other);
}
@Override
@@ -276,21 +289,27 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- super.merge(other);
- DoubleStatisticsImpl dbl = (DoubleStatisticsImpl) other;
- if (!hasMinimum) {
- hasMinimum = dbl.hasMinimum;
- minimum = dbl.minimum;
- maximum = dbl.maximum;
- } else if (dbl.hasMinimum) {
- if (dbl.minimum < minimum) {
+ if (other instanceof DoubleStatisticsImpl) {
+ DoubleStatisticsImpl dbl = (DoubleStatisticsImpl) other;
+ if (!hasMinimum) {
+ hasMinimum = dbl.hasMinimum;
minimum = dbl.minimum;
- }
- if (dbl.maximum > maximum) {
maximum = dbl.maximum;
+ } else if (dbl.hasMinimum) {
+ if (dbl.minimum < minimum) {
+ minimum = dbl.minimum;
+ }
+ if (dbl.maximum > maximum) {
+ maximum = dbl.maximum;
+ }
+ }
+ sum += dbl.sum;
+ } else {
+ if (isStatsExists() && hasMinimum) {
+ throw new IllegalArgumentException("Incompatible merging of double column statistics");
}
}
- sum += dbl.sum;
+ super.merge(other);
}
@Override
@@ -382,25 +401,31 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- super.merge(other);
- StringStatisticsImpl str = (StringStatisticsImpl) other;
- if (minimum == null) {
- if(str.minimum != null) {
- maximum = new Text(str.getMaximum());
- minimum = new Text(str.getMinimum());
- } else {
+ if (other instanceof StringStatisticsImpl) {
+ StringStatisticsImpl str = (StringStatisticsImpl) other;
+ if (minimum == null) {
+ if (str.minimum != null) {
+ maximum = new Text(str.getMaximum());
+ minimum = new Text(str.getMinimum());
+ } else {
/* both are empty */
- maximum = minimum = null;
+ maximum = minimum = null;
+ }
+ } else if (str.minimum != null) {
+ if (minimum.compareTo(str.minimum) > 0) {
+ minimum = new Text(str.getMinimum());
+ }
+ if (maximum.compareTo(str.maximum) < 0) {
+ maximum = new Text(str.getMaximum());
+ }
}
- } else if (str.minimum != null) {
- if (minimum.compareTo(str.minimum) > 0) {
- minimum = new Text(str.getMinimum());
- }
- if (maximum.compareTo(str.maximum) < 0) {
- maximum = new Text(str.getMaximum());
+ sum += str.sum;
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of string column statistics");
}
}
- sum += str.sum;
+ super.merge(other);
}
@Override
@@ -476,9 +501,15 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
+ if (other instanceof BinaryColumnStatistics) {
+ BinaryStatisticsImpl bin = (BinaryStatisticsImpl) other;
+ sum += bin.sum;
+ } else {
+ if (isStatsExists() && sum != 0) {
+ throw new IllegalArgumentException("Incompatible merging of binary column statistics");
+ }
+ }
super.merge(other);
- BinaryStatisticsImpl bin = (BinaryStatisticsImpl) other;
- sum += bin.sum;
}
@Override
@@ -556,25 +587,31 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- super.merge(other);
- DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other;
- if (minimum == null) {
- minimum = dec.minimum;
- maximum = dec.maximum;
- sum = dec.sum;
- } else if (dec.minimum != null) {
- if (minimum.compareTo(dec.minimum) > 0) {
+ if (other instanceof DecimalStatisticsImpl) {
+ DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other;
+ if (minimum == null) {
minimum = dec.minimum;
- }
- if (maximum.compareTo(dec.maximum) < 0) {
maximum = dec.maximum;
+ sum = dec.sum;
+ } else if (dec.minimum != null) {
+ if (minimum.compareTo(dec.minimum) > 0) {
+ minimum = dec.minimum;
+ }
+ if (maximum.compareTo(dec.maximum) < 0) {
+ maximum = dec.maximum;
+ }
+ if (sum == null || dec.sum == null) {
+ sum = null;
+ } else {
+ sum = sum.add(dec.sum);
+ }
}
- if (sum == null || dec.sum == null) {
- sum = null;
- } else {
- sum = sum.add(dec.sum);
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of decimal column statistics");
}
}
+ super.merge(other);
}
@Override
@@ -582,7 +619,7 @@ class ColumnStatisticsImpl implements ColumnStatistics {
OrcProto.ColumnStatistics.Builder result = super.serialize();
OrcProto.DecimalStatistics.Builder dec =
OrcProto.DecimalStatistics.newBuilder();
- if (getNumberOfValues() != 0) {
+ if (getNumberOfValues() != 0 && minimum != null) {
dec.setMinimum(minimum.toString());
dec.setMaximum(maximum.toString());
}
@@ -666,19 +703,25 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- super.merge(other);
- DateStatisticsImpl dateStats = (DateStatisticsImpl) other;
- if (minimum == null) {
- minimum = dateStats.minimum;
- maximum = dateStats.maximum;
- } else if (dateStats.minimum != null) {
- if (minimum > dateStats.minimum) {
+ if (other instanceof DateStatisticsImpl) {
+ DateStatisticsImpl dateStats = (DateStatisticsImpl) other;
+ if (minimum == null) {
minimum = dateStats.minimum;
- }
- if (maximum < dateStats.maximum) {
maximum = dateStats.maximum;
+ } else if (dateStats.minimum != null) {
+ if (minimum > dateStats.minimum) {
+ minimum = dateStats.minimum;
+ }
+ if (maximum < dateStats.maximum) {
+ maximum = dateStats.maximum;
+ }
+ }
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of date column statistics");
}
}
+ super.merge(other);
}
@Override
@@ -686,7 +729,7 @@ class ColumnStatisticsImpl implements ColumnStatistics {
OrcProto.ColumnStatistics.Builder result = super.serialize();
OrcProto.DateStatistics.Builder dateStats =
OrcProto.DateStatistics.newBuilder();
- if (getNumberOfValues() != 0) {
+ if (getNumberOfValues() != 0 && minimum != null) {
dateStats.setMinimum(minimum);
dateStats.setMaximum(maximum);
}
@@ -769,19 +812,25 @@ class ColumnStatisticsImpl implements ColumnStatistics {
@Override
void merge(ColumnStatisticsImpl other) {
- super.merge(other);
- TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl) other;
- if (minimum == null) {
- minimum = timestampStats.minimum;
- maximum = timestampStats.maximum;
- } else if (timestampStats.minimum != null) {
- if (minimum > timestampStats.minimum) {
+ if (other instanceof TimestampStatisticsImpl) {
+ TimestampStatisticsImpl timestampStats = (TimestampStatisticsImpl) other;
+ if (minimum == null) {
minimum = timestampStats.minimum;
- }
- if (maximum < timestampStats.maximum) {
maximum = timestampStats.maximum;
+ } else if (timestampStats.minimum != null) {
+ if (minimum > timestampStats.minimum) {
+ minimum = timestampStats.minimum;
+ }
+ if (maximum < timestampStats.maximum) {
+ maximum = timestampStats.maximum;
+ }
+ }
+ } else {
+ if (isStatsExists() && minimum != null) {
+ throw new IllegalArgumentException("Incompatible merging of timestamp column statistics");
}
}
+ super.merge(other);
}
@Override
@@ -789,7 +838,7 @@ class ColumnStatisticsImpl implements ColumnStatistics {
OrcProto.ColumnStatistics.Builder result = super.serialize();
OrcProto.TimestampStatistics.Builder timestampStats = OrcProto.TimestampStatistics
.newBuilder();
- if (getNumberOfValues() != 0) {
+ if (getNumberOfValues() != 0 && minimum != null) {
timestampStats.setMinimum(minimum);
timestampStats.setMaximum(maximum);
}
@@ -878,6 +927,10 @@ class ColumnStatisticsImpl implements ColumnStatistics {
throw new UnsupportedOperationException("Can't update timestamp");
}
+ boolean isStatsExists() {
+ return (count > 0 || hasNull == true);
+ }
+
void merge(ColumnStatisticsImpl stats) {
count += stats.count;
hasNull |= stats.hasNull;
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
index 11f05c6..a62fc1e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileKeyWrapper.java
@@ -18,26 +18,35 @@
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.WritableComparable;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.WritableComparable;
+
/**
* Key for OrcFileMergeMapper task. Contains orc file related information that
* should match before merging two orc files.
*/
public class OrcFileKeyWrapper implements WritableComparable<OrcFileKeyWrapper> {
- protected Path inputPath;
- protected CompressionKind compression;
- protected long compressBufferSize;
- protected List<OrcProto.Type> types;
- protected int rowIndexStride;
- protected OrcFile.Version version;
+ private Path inputPath;
+ private CompressionKind compression;
+ private long compressBufferSize;
+ private List<OrcProto.Type> types;
+ private int rowIndexStride;
+ private OrcFile.Version version;
+ private boolean isIncompatFile;
+
+ public boolean isIncompatFile() {
+ return isIncompatFile;
+ }
+
+ public void setIsIncompatFile(boolean isIncompatFile) {
+ this.isIncompatFile = isIncompatFile;
+ }
public OrcFile.Version getVersion() {
return version;
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
index cf6fa2a..41a97a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
@@ -18,19 +18,18 @@
package org.apache.hadoop.hive.ql.io.orc;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
public class OrcFileStripeMergeRecordReader implements
RecordReader<OrcFileKeyWrapper, OrcFileValueWrapper> {
-
private final Reader reader;
private final Path path;
protected Iterator<StripeInformation> iter;
@@ -38,6 +37,7 @@ public class OrcFileStripeMergeRecordReader implements
private int stripeIdx;
private long start;
private long end;
+ private boolean skipFile;
public OrcFileStripeMergeRecordReader(Configuration conf, FileSplit split) throws IOException {
path = split.getPath();
@@ -68,11 +68,23 @@ public class OrcFileStripeMergeRecordReader implements
@Override
public boolean next(OrcFileKeyWrapper key, OrcFileValueWrapper value) throws IOException {
+ if (skipFile) {
+ return false;
+ }
return nextStripe(key, value);
}
protected boolean nextStripe(OrcFileKeyWrapper keyWrapper, OrcFileValueWrapper valueWrapper)
throws IOException {
+ // missing stripe stats (old format). If numRows is 0 then its an empty file and no statistics
+ // is present. We have to differentiate no stats (empty file) vs missing stats (old format).
+ if ((stripeStatistics == null || stripeStatistics.isEmpty()) && reader.getNumberOfRows() > 0) {
+ keyWrapper.setInputPath(path);
+ keyWrapper.setIsIncompatFile(true);
+ skipFile = true;
+ return true;
+ }
+
while (iter.hasNext()) {
StripeInformation si = iter.next();
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/test/queries/clientpositive/orc_merge9.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/orc_merge9.q b/ql/src/test/queries/clientpositive/orc_merge9.q
new file mode 100644
index 0000000..010b5a1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/orc_merge9.q
@@ -0,0 +1,44 @@
+create table ts_merge (
+userid bigint,
+string1 string,
+subtype double,
+decimal1 decimal(38,18),
+ts timestamp
+) stored as orc;
+
+load data local inpath '../../data/files/orc_split_elim.orc' overwrite into table ts_merge;
+load data local inpath '../../data/files/orc_split_elim.orc' into table ts_merge;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/ts_merge/;
+
+set hive.merge.orcfile.stripe.level=true;
+set hive.merge.tezfiles=true;
+set hive.merge.mapfiles=true;
+set hive.merge.mapredfiles=true;
+
+select count(*) from ts_merge;
+alter table ts_merge concatenate;
+select count(*) from ts_merge;
+
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/ts_merge/;
+
+-- incompatible merge test (stripe statistics missing)
+
+create table a_merge like alltypesorc;
+
+insert overwrite table a_merge select * from alltypesorc;
+load data local inpath '../../data/files/alltypesorc' into table a_merge;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/a_merge/;
+
+select count(*) from a_merge;
+alter table a_merge concatenate;
+select count(*) from a_merge;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/a_merge/;
+
+insert into table a_merge select * from alltypesorc;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/a_merge/;
+
+select count(*) from a_merge;
+alter table a_merge concatenate;
+select count(*) from a_merge;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/a_merge/;
http://git-wip-us.apache.org/repos/asf/hive/blob/3f8b0ef8/ql/src/test/results/clientpositive/orc_merge9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/orc_merge9.q.out b/ql/src/test/results/clientpositive/orc_merge9.q.out
new file mode 100644
index 0000000..bdf0fd3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/orc_merge9.q.out
@@ -0,0 +1,186 @@
+PREHOOK: query: create table ts_merge (
+userid bigint,
+string1 string,
+subtype double,
+decimal1 decimal(38,18),
+ts timestamp
+) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ts_merge
+POSTHOOK: query: create table ts_merge (
+userid bigint,
+string1 string,
+subtype double,
+decimal1 decimal(38,18),
+ts timestamp
+) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ts_merge
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' overwrite into table ts_merge
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@ts_merge
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' overwrite into table ts_merge
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@ts_merge
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table ts_merge
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@ts_merge
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table ts_merge
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@ts_merge
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from ts_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ts_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from ts_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ts_merge
+#### A masked pattern was here ####
+50000
+PREHOOK: query: alter table ts_merge concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@ts_merge
+PREHOOK: Output: default@ts_merge
+POSTHOOK: query: alter table ts_merge concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@ts_merge
+POSTHOOK: Output: default@ts_merge
+PREHOOK: query: select count(*) from ts_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ts_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from ts_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ts_merge
+#### A masked pattern was here ####
+50000
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: query: -- incompatible merge test (stripe statistics missing)
+
+create table a_merge like alltypesorc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: -- incompatible merge test (stripe statistics missing)
+
+create table a_merge like alltypesorc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@a_merge
+PREHOOK: query: insert overwrite table a_merge select * from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: insert overwrite table a_merge select * from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@a_merge
+POSTHOOK: Lineage: a_merge.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: a_merge.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: a_merge.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: a_merge.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: a_merge.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: a_merge.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: a_merge.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: a_merge.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: a_merge.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+PREHOOK: query: load data local inpath '../../data/files/alltypesorc' into table a_merge
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: load data local inpath '../../data/files/alltypesorc' into table a_merge
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@a_merge
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from a_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from a_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+24576
+PREHOOK: query: alter table a_merge concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@a_merge
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: alter table a_merge concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@a_merge
+POSTHOOK: Output: default@a_merge
+PREHOOK: query: select count(*) from a_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from a_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+24576
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: insert into table a_merge select * from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: insert into table a_merge select * from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: default@a_merge
+POSTHOOK: Lineage: a_merge.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ]
+POSTHOOK: Lineage: a_merge.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ]
+POSTHOOK: Lineage: a_merge.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ]
+POSTHOOK: Lineage: a_merge.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ]
+POSTHOOK: Lineage: a_merge.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ]
+POSTHOOK: Lineage: a_merge.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ]
+POSTHOOK: Lineage: a_merge.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
+POSTHOOK: Lineage: a_merge.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ]
+POSTHOOK: Lineage: a_merge.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ]
+POSTHOOK: Lineage: a_merge.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ]
+Found 3 items
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from a_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from a_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+36864
+PREHOOK: query: alter table a_merge concatenate
+PREHOOK: type: ALTER_TABLE_MERGE
+PREHOOK: Input: default@a_merge
+PREHOOK: Output: default@a_merge
+POSTHOOK: query: alter table a_merge concatenate
+POSTHOOK: type: ALTER_TABLE_MERGE
+POSTHOOK: Input: default@a_merge
+POSTHOOK: Output: default@a_merge
+PREHOOK: query: select count(*) from a_merge
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from a_merge
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a_merge
+#### A masked pattern was here ####
+36864
+Found 2 items
+#### A masked pattern was here ####
[27/61] [abbrv] hive git commit: HIVE-10979: Fix failed tests in
TestSchemaTool after the version number change in HIVE-10921(Ferdinand Xu,
reviewed by Sergey Shelukhin)
Posted by xu...@apache.org.
HIVE-10979: Fix failed tests in TestSchemaTool after the version number change in HIVE-10921(Ferdinand Xu, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/670f57c1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/670f57c1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/670f57c1
Branch: refs/heads/spark
Commit: 670f57c188b6f02f2c7f55b758ff403501058285
Parents: 77978f3
Author: Ferdinand Xu <ch...@intel.com>
Authored: Sun Jun 14 13:44:40 2015 -0400
Committer: Ferdinand Xu <ch...@intel.com>
Committed: Sun Jun 14 13:44:40 2015 -0400
----------------------------------------------------------------------
metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql | 2 +-
.../scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql | 4 ++--
metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql | 2 +-
metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql | 2 +-
.../scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql | 6 +++---
metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql | 2 +-
.../scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql | 6 +++---
.../scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql | 2 +-
.../upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql | 6 +++---
9 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
index c50375f..1cc0a24 100644
--- a/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
@@ -332,5 +332,5 @@ RUN 'hive-txn-schema-0.13.0.derby.sql';
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '1.3.0', 'Hive release version 1.3.0');
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '2.0.0', 'Hive release version 2.0.0');
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
index dee744b..976ebd1 100644
--- a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
@@ -1,3 +1,3 @@
--- Upgrade MetaStore schema from 1.2.0 to 1.3.0
+-- Upgrade MetaStore schema from 1.3.0 to 2.0.0
-UPDATE "APP".VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
+UPDATE "APP".VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
index 7165edd..36e7c1f 100644
--- a/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
@@ -944,4 +944,4 @@ ALTER TABLE TXN_COMPONENTS WITH CHECK ADD FOREIGN KEY(TC_TXNID) REFERENCES TXNS
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '1.3.0', 'Hive release version 1.3.0');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '2.0.0', 'Hive release version 2.0.0');
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
index bc63f0d..3a595dd 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
@@ -824,7 +824,7 @@ SOURCE hive-txn-schema-0.13.0.mysql.sql;
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '1.3.0', 'Hive release version 1.3.0');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '2.0.0', 'Hive release version 2.0.0');
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
index d5a31f7..22d5242 100644
--- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
-UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
-SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
+SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
+UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
+SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
index 7605bc7..8d963ce 100644
--- a/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
@@ -785,4 +785,4 @@ CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID);
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '1.3.0', 'Hive release version 1.3.0');
+INSERT INTO VERSION (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES (1, '2.0.0', 'Hive release version 2.0.0');
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
index f072a1c..efff2c9 100644
--- a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
@@ -1,4 +1,4 @@
-SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual;
+SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual;
-UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
-SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual;
+UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
+SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
index 3ab5b3e..6442eb1 100644
--- a/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
@@ -1454,4 +1454,4 @@ GRANT ALL ON SCHEMA public TO PUBLIC;
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO "VERSION" ("VER_ID", "SCHEMA_VERSION", "VERSION_COMMENT") VALUES (1, '1.3.0', 'Hive release version 1.3.0');
+INSERT INTO "VERSION" ("VER_ID", "SCHEMA_VERSION", "VERSION_COMMENT") VALUES (1, '2.0.0', 'Hive release version 2.0.0');
http://git-wip-us.apache.org/repos/asf/hive/blob/670f57c1/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
index cdd3792..b6b0c35 100644
--- a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
@@ -1,8 +1,8 @@
-SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0';
+SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0';
-UPDATE "VERSION" SET "SCHEMA_VERSION"='1.3.0', "VERSION_COMMENT"='Hive release version 1.3.0' where "VER_ID"=1;
-SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0';
+UPDATE "VERSION" SET "SCHEMA_VERSION"='2.0.0', "VERSION_COMMENT"='Hive release version 2.0.0' where "VER_ID"=1;
+SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0';
--
[42/61] [abbrv] hive git commit: HIVE-11006 improve logging wrt ACID
module (Eugene Koifman, reviewed by Alan Gates)
Posted by xu...@apache.org.
HIVE-11006 improve logging wrt ACID module (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37e82baa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37e82baa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37e82baa
Branch: refs/heads/spark
Commit: 37e82baafbdf171757cd07939ee9745e4c3b5adb
Parents: a3792b7
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Wed Jun 17 13:05:10 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Wed Jun 17 13:05:10 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/txn/CompactionInfo.java | 9 +++++
.../metastore/txn/CompactionTxnHandler.java | 19 +++++------
.../hadoop/hive/metastore/txn/TxnHandler.java | 36 +++++++++++---------
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 4 +--
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 2 +-
5 files changed, 41 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/37e82baa/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
index 939df3f..1dae7b9 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
@@ -73,4 +73,13 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
public int compareTo(CompactionInfo o) {
return getFullPartitionName().compareTo(o.getFullPartitionName());
}
+ public String toString() {
+ return "id:" + id + "," +
+ "dbname:" + dbname + "," +
+ "tableName:" + tableName + "," +
+ "partName:" + partName + "," +
+ "type:" + type + "," +
+ "runAs:" + runAs + "," +
+ "tooManyAborts:" + tooManyAborts;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/37e82baa/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 26e72be..328a65c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -95,7 +95,7 @@ public class CompactionTxnHandler extends TxnHandler {
dbConn.rollback();
} catch (SQLException e) {
LOG.error("Unable to connect to transaction database " + e.getMessage());
- checkRetryable(dbConn, e, "findPotentialCompactions");
+ checkRetryable(dbConn, e, "findPotentialCompactions(maxAborted:" + maxAborted + ")");
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
@@ -133,7 +133,7 @@ public class CompactionTxnHandler extends TxnHandler {
LOG.error("Unable to update compaction queue, " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "setRunAs");
+ checkRetryable(dbConn, e, "setRunAs(cq_id:" + cq_id + ",user:" + user +")");
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
@@ -194,7 +194,7 @@ public class CompactionTxnHandler extends TxnHandler {
LOG.error("Unable to select next element for compaction, " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "findNextToCompact");
+ checkRetryable(dbConn, e, "findNextToCompact(workerId:" + workerId + ")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -232,7 +232,7 @@ public class CompactionTxnHandler extends TxnHandler {
LOG.error("Unable to update compaction queue " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "markCompacted");
+ checkRetryable(dbConn, e, "markCompacted(" + info + ")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -374,7 +374,7 @@ public class CompactionTxnHandler extends TxnHandler {
LOG.error("Unable to delete from compaction queue " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "markCleaned");
+ checkRetryable(dbConn, e, "markCleaned(" + info + ")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -465,7 +465,7 @@ public class CompactionTxnHandler extends TxnHandler {
e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "revokeFromLocalWorkers");
+ checkRetryable(dbConn, e, "revokeFromLocalWorkers(hostname:" + hostname +")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -508,7 +508,7 @@ public class CompactionTxnHandler extends TxnHandler {
e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "revokeTimedoutWorkers");
+ checkRetryable(dbConn, e, "revokeTimedoutWorkers(timeout:" + timeout + ")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -565,10 +565,9 @@ public class CompactionTxnHandler extends TxnHandler {
dbConn.commit();
return columns;
} catch (SQLException e) {
- LOG.error("Failed to find columns to analyze stats on for " + ci.tableName +
- (ci.partName == null ? "" : "/" + ci.partName), e);
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "findColumnsWithStats");
+ checkRetryable(dbConn, e, "findColumnsWithStats(" + ci.tableName +
+ (ci.partName == null ? "" : "/" + ci.partName) + ")");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/37e82baa/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 7c3b55c..fd9c275 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -318,7 +318,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "openTxns");
+ checkRetryable(dbConn, e, "openTxns(" + rqst + ")");
throw new MetaException("Unable to select from transaction database "
+ StringUtils.stringifyException(e));
} finally {
@@ -347,7 +347,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "abortTxn");
+ checkRetryable(dbConn, e, "abortTxn(" + rqst + ")");
throw new MetaException("Unable to update transaction database "
+ StringUtils.stringifyException(e));
} finally {
@@ -400,7 +400,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "commitTxn");
+ checkRetryable(dbConn, e, "commitTxn(" + rqst + ")");
throw new MetaException("Unable to update transaction database "
+ StringUtils.stringifyException(e));
} finally {
@@ -423,7 +423,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "lock");
+ checkRetryable(dbConn, e, "lock(" + rqst + ")");
throw new MetaException("Unable to update transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -444,7 +444,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "lockNoWait");
+ checkRetryable(dbConn, e, "lockNoWait(" + rqst + ")");
throw new MetaException("Unable to update transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -475,7 +475,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "checkLock");
+ checkRetryable(dbConn, e, "checkLock(" + rqst + " )");
throw new MetaException("Unable to update transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -526,7 +526,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "unlock");
+ checkRetryable(dbConn, e, "unlock(" + rqst + ")");
throw new MetaException("Unable to update transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -593,7 +593,7 @@ public class TxnHandler {
LOG.debug("Going to rollback");
dbConn.rollback();
} catch (SQLException e) {
- checkRetryable(dbConn, e, "showLocks");
+ checkRetryable(dbConn, e, "showLocks(" + rqst + ")");
throw new MetaException("Unable to select from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -624,7 +624,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "heartbeat");
+ checkRetryable(dbConn, e, "heartbeat(" + ids + ")");
throw new MetaException("Unable to select from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -661,7 +661,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "heartbeatTxnRange");
+ checkRetryable(dbConn, e, "heartbeatTxnRange(" + rqst + ")");
throw new MetaException("Unable to select from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -742,7 +742,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "compact");
+ checkRetryable(dbConn, e, "compact(" + rqst + ")");
throw new MetaException("Unable to select from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -792,7 +792,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "showCompact");
+ checkRetryable(dbConn, e, "showCompact(" + rqst + ")");
throw new MetaException("Unable to select from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -835,7 +835,7 @@ public class TxnHandler {
} catch (SQLException e) {
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "addDynamicPartitions");
+ checkRetryable(dbConn, e, "addDynamicPartitions(" + rqst + ")");
throw new MetaException("Unable to insert into from transaction database " +
StringUtils.stringifyException(e));
} finally {
@@ -955,6 +955,7 @@ public class TxnHandler {
* @param caller name of the method calling this
* @throws org.apache.hadoop.hive.metastore.txn.TxnHandler.RetryException when deadlock
* detected and retry count has not been exceeded.
+ * TODO: make "caller" more elaborate like include lockId for example
*/
protected void checkRetryable(Connection conn,
SQLException e,
@@ -977,10 +978,12 @@ public class TxnHandler {
(dbProduct == DatabaseProduct.ORACLE && (e.getMessage().contains("deadlock detected")
|| e.getMessage().contains("can't serialize access for this transaction")))) {
if (deadlockCnt++ < ALLOWED_REPEATED_DEADLOCKS) {
- LOG.warn("Deadlock detected in " + caller + ", trying again.");
+ long waitInterval = deadlockRetryInterval * deadlockCnt;
+ LOG.warn("Deadlock detected in " + caller + ". Will wait " + waitInterval +
+ "ms try again up to " + (ALLOWED_REPEATED_DEADLOCKS - deadlockCnt + 1) + " times.");
// Pause for a just a bit for retrying to avoid immediately jumping back into the deadlock.
try {
- Thread.sleep(deadlockRetryInterval * deadlockCnt);
+ Thread.sleep(waitInterval);
} catch (InterruptedException ie) {
// NOP
}
@@ -993,13 +996,14 @@ public class TxnHandler {
else if(isRetryable(e)) {
//in MSSQL this means Communication Link Failure
if(retryNum++ < retryLimit) {
+ LOG.warn("Retryable error detected in " + caller + ". Will wait " + retryInterval +
+ "ms and retry up to " + (retryLimit - retryNum + 1) + " times. Error: " + getMessage(e));
try {
Thread.sleep(retryInterval);
}
catch(InterruptedException ex) {
//
}
- LOG.warn("Retryable error detected in " + caller + ", trying again: " + getMessage(e));
throw new RetryException();
}
else {
http://git-wip-us.apache.org/repos/asf/hive/blob/37e82baa/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index e8c49ef..aa00573 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -79,10 +79,10 @@ public class DbLockManager implements HiveLockManager{
*/
LockState lock(LockRequest lock, String queryId, boolean isBlocking, List<HiveLock> acquiredLocks) throws LockException {
try {
- LOG.debug("Requesting: queryId=" + queryId + " " + lock);
+ LOG.info("Requesting: queryId=" + queryId + " " + lock);
LockResponse res = client.lock(lock);
//link lockId to queryId
- LOG.debug("Response " + res);
+ LOG.info("Response to queryId=" + queryId + " " + res);
if(!isBlocking) {
if(res.getState() == LockState.WAITING) {
return LockState.WAITING;
http://git-wip-us.apache.org/repos/asf/hive/blob/37e82baa/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index ccbac80..f8fff1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -105,7 +105,7 @@ public class DbTxnManager extends HiveTxnManagerImpl {
LockRequestBuilder rqstBuilder = new LockRequestBuilder();
//link queryId to txnId
- LOG.debug("Setting lock request transaction to " + txnId + " for queryId=" + plan.getQueryId());
+ LOG.info("Setting lock request transaction to " + txnId + " for queryId=" + plan.getQueryId());
rqstBuilder.setTransactionId(txnId)
.setUser(username);
[56/61] [abbrv] hive git commit: HIVE-11050:
testCliDriver_vector_outer_join.* failures in Unit tests due to unstable data
creation queries (Matt McCline reviewed by Gunther Hagleitner)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b97303cd/ql/src/test/results/clientpositive/vector_outer_join4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/vector_outer_join4.q.out
index c07b1d6..50ae497 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join4.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join4.q.out
@@ -1,43 +1,107 @@
-PREHOOK: query: create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null limit 10
+PREHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc1b
-POSTHOOK: query: create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null limit 10
+POSTHOOK: query: -- Using cint and ctinyint in test queries
+create table small_alltypesorc1b as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc1b
-PREHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null limit 10
+PREHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc2b
-POSTHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null limit 10
+POSTHOOK: query: create table small_alltypesorc2b as select * from alltypesorc where cint is null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc2b
-PREHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null limit 10
+PREHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc3b
-POSTHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null limit 10
+POSTHOOK: query: create table small_alltypesorc3b as select * from alltypesorc where cint is not null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc3b
-PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null limit 10
+PREHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: database:default
PREHOOK: Output: default@small_alltypesorc4b
-POSTHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null limit 10
+POSTHOOK: query: create table small_alltypesorc4b as select * from alltypesorc where cint is null and ctinyint is null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 10
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@small_alltypesorc4b
+PREHOOK: query: select * from small_alltypesorc1b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc1b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc1b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc1b
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -3097 253665376 NULL -64.0 -3097.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.013 1969-12-31 16:00:06.097 true NULL
+-64 -3586 626923679 NULL -64.0 -3586.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.952 1969-12-31 15:59:51.131 true NULL
+-64 -4018 626923679 NULL -64.0 -4018.0 821UdmGbkEf4j NULL 1969-12-31 15:59:58.959 1969-12-31 16:00:07.803 true NULL
+-64 -4040 528534767 NULL -64.0 -4040.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:47.733 1969-12-31 15:59:46.044 true NULL
+-64 -4803 626923679 NULL -64.0 -4803.0 821UdmGbkEf4j NULL 1969-12-31 16:00:04.662 1969-12-31 16:00:01.609 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+PREHOOK: query: select * from small_alltypesorc2b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc2b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc2b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc2b
+#### A masked pattern was here ####
+-64 -200 NULL -1809444706 -64.0 -200.0 NULL B87YVb3UASqg 1969-12-31 16:00:10.858 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 2118653994 -64.0 -200.0 NULL ONHGSDy1U4Ft431DfQp15 1969-12-31 16:00:03.944 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 927647669 -64.0 -200.0 NULL DhxkBT 1969-12-31 16:00:00.199 1969-12-31 15:59:55.451 NULL false
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1090418478 -64.0 -7196.0 NULL 3E06w2 1969-12-31 16:00:00.29 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1805860756 -64.0 -7196.0 NULL 4aOn4s2ATygu0476eD 1969-12-31 16:00:12.339 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+PREHOOK: query: select * from small_alltypesorc3b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc3b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc3b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc3b
+#### A masked pattern was here ####
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -609074876 -1887561756 NULL NULL EcM71 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:55.061 true false
+NULL NULL -700300206 -1887561756 NULL NULL kdqQE010 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:58.384 false false
+NULL NULL -726473298 1864027286 NULL NULL OFy1a1xf37f75b5N 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:11.799 true true
+NULL NULL -738747840 -1645852809 NULL NULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:11.55 true false
+NULL NULL -838810013 1864027286 NULL NULL N016jPED08o 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:44.252 false true
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
+PREHOOK: query: select * from small_alltypesorc4b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc4b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc4b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc4b
+#### A masked pattern was here ####
PREHOOK: query: create table small_alltypesorc_b stored as orc as select * from
(select * from (select * from small_alltypesorc1b) sq1
union all
@@ -84,6 +148,44 @@ POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMN
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
+PREHOOK: query: select * from small_alltypesorc_b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_alltypesorc_b
+#### A masked pattern was here ####
+POSTHOOK: query: select * from small_alltypesorc_b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_alltypesorc_b
+#### A masked pattern was here ####
+-64 -10462 626923679 NULL -64.0 -10462.0 821UdmGbkEf4j NULL 1969-12-31 16:00:02.496 1969-12-31 16:00:00.164 true NULL
+-64 -15920 528534767 NULL -64.0 -15920.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:51.859 1969-12-31 16:00:14.468 true NULL
+-64 -200 NULL -1809444706 -64.0 -200.0 NULL B87YVb3UASqg 1969-12-31 16:00:10.858 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 2118653994 -64.0 -200.0 NULL ONHGSDy1U4Ft431DfQp15 1969-12-31 16:00:03.944 1969-12-31 15:59:55.451 NULL true
+-64 -200 NULL 927647669 -64.0 -200.0 NULL DhxkBT 1969-12-31 16:00:00.199 1969-12-31 15:59:55.451 NULL false
+-64 -3097 253665376 NULL -64.0 -3097.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.013 1969-12-31 16:00:06.097 true NULL
+-64 -3586 626923679 NULL -64.0 -3586.0 821UdmGbkEf4j NULL 1969-12-31 16:00:11.952 1969-12-31 15:59:51.131 true NULL
+-64 -4018 626923679 NULL -64.0 -4018.0 821UdmGbkEf4j NULL 1969-12-31 15:59:58.959 1969-12-31 16:00:07.803 true NULL
+-64 -4040 528534767 NULL -64.0 -4040.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:47.733 1969-12-31 15:59:46.044 true NULL
+-64 -4803 626923679 NULL -64.0 -4803.0 821UdmGbkEf4j NULL 1969-12-31 16:00:04.662 1969-12-31 16:00:01.609 true NULL
+-64 -6907 253665376 NULL -64.0 -6907.0 1cGVWH7n1QU NULL NULL 1969-12-31 15:59:53.66 true NULL
+-64 -7196 NULL -1615920595 -64.0 -7196.0 NULL X5rDjl 1969-12-31 16:00:11.912 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -1639157869 -64.0 -7196.0 NULL IJ0Oj7qAiqNGsN7gn 1969-12-31 16:00:01.785 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL -527203677 -64.0 -7196.0 NULL JBE4H5RoK412Cs260I72 1969-12-31 15:59:50.184 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1090418478 -64.0 -7196.0 NULL 3E06w2 1969-12-31 16:00:00.29 1969-12-31 15:59:58.174 NULL true
+-64 -7196 NULL 1805860756 -64.0 -7196.0 NULL 4aOn4s2ATygu0476eD 1969-12-31 16:00:12.339 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 406535485 -64.0 -7196.0 NULL E011i 1969-12-31 15:59:56.048 1969-12-31 15:59:58.174 NULL false
+-64 -7196 NULL 658026952 -64.0 -7196.0 NULL 4tAur 1969-12-31 15:59:53.866 1969-12-31 15:59:58.174 NULL true
+-64 -8080 528534767 NULL -64.0 -8080.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:58.044 1969-12-31 15:59:48.655 true NULL
+-64 -9842 253665376 NULL -64.0 -9842.0 1cGVWH7n1QU NULL 1969-12-31 16:00:00.631 1969-12-31 16:00:01.781 true NULL
+NULL NULL -1015272448 -1887561756 NULL NULL jTQ68531mP 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:45.854 false false
+NULL NULL -609074876 -1887561756 NULL NULL EcM71 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:55.061 true false
+NULL NULL -700300206 -1887561756 NULL NULL kdqQE010 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 15:59:58.384 false false
+NULL NULL -726473298 1864027286 NULL NULL OFy1a1xf37f75b5N 4KWs6gw7lv2WYd66P NULL 1969-12-31 16:00:11.799 true true
+NULL NULL -738747840 -1645852809 NULL NULL vmAT10eeE47fgH20pLi xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:11.55 true false
+NULL NULL -838810013 1864027286 NULL NULL N016jPED08o 4KWs6gw7lv2WYd66P NULL 1969-12-31 15:59:44.252 false true
+NULL NULL -850295959 -1887561756 NULL NULL WMIgGA73 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:00.348 false false
+NULL NULL -886426182 -1887561756 NULL NULL 0i88xYq3gx1nW4vKjp7vBp3 4hA4KQj2vD3fI6gX82220d NULL 1969-12-31 16:00:04.472 true false
+NULL NULL -899422227 -1645852809 NULL NULL 73xdw4X xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:07.395 false false
+NULL NULL -971543377 -1645852809 NULL NULL uN803aW xH7445Rals48VOulSyR5F NULL 1969-12-31 16:00:05.43 false false
PREHOOK: query: explain
select *
from small_alltypesorc_b c
@@ -112,11 +214,11 @@ STAGE PLANS:
$hdt$_1:c
TableScan
alias: c
- Statistics: Num rows: 30 Data size: 5670 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6680 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 30 Data size: 5670 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6680 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
0 _col2 (type: int)
@@ -127,11 +229,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: c
- Statistics: Num rows: 30 Data size: 5670 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6680 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 30 Data size: 5670 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 30 Data size: 6680 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join0 to 1
@@ -139,10 +241,10 @@ STAGE PLANS:
0 _col2 (type: int)
1 _col2 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
- Statistics: Num rows: 33 Data size: 6237 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 7348 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 33 Data size: 6237 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 33 Data size: 7348 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -175,416 +277,60 @@ left outer join small_alltypesorc_b cd
POSTHOOK: type: QUERY
POSTHOOK: Input: default@small_alltypesorc_b
#### A masked pattern was here ####
--10 -200 NULL -1818374653 -10.0 -200.0 NULL uFavNs7g58qrfyCH681d 1969-12-31 16:00:03.248 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--21 -200 NULL NULL -21.0 -200.0 NULL NULL 1969-12-31 16:00:09.052 1969-12-31 15:59:55.451 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--30 -200 NULL -815881183 -30.0 -200.0 NULL B0B5kG3OIl6C 1969-12-31 15:59:44.842 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--39 -200 NULL 944477914 -39.0 -200.0 NULL T8brJ213nd7rhW8XdnB1 1969-12-31 16:00:00.958 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
--50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
--63 -200 NULL 1927856372 -63.0 -200.0 NULL v6mk2b7oX 1969-12-31 16:00:06.852 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-3 -200 NULL -1438142492 3.0 -200.0 NULL T8Uakh8tudd1XRG5yKW8Y42H 1969-12-31 16:00:07.648 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-31 -200 NULL -245476731 31.0 -200.0 NULL 3E3BxP 1969-12-31 15:59:54.739 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-38 -200 NULL -1090414113 38.0 -200.0 NULL 5oQ43l 1969-12-31 16:00:05.478 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-38 -200 NULL 1576772382 38.0 -200.0 NULL X3Ufbt46AUIfHe 1969-12-31 15:59:49.567 1969-12-31 15:59:55.451 NULL true NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-8 -200 NULL -1236645308 8.0 -200.0 NULL M28wJxOvunO3EImapE7OApQ 1969-12-31 15:59:46.007 1969-12-31 15:59:55.451 NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.151 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 13889 528534767 NULL NULL 13889.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:00.423 true NULL
-NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL NULL 3321 528534767 NULL NULL 3321.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:52.967 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -11 -15431 528534767 NULL -11.0 -15431.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:52.176 1969-12-31 16:00:07.787 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -28 -15813 528534767 NULL -28.0 -15813.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:55.787 1969-12-31 16:00:01.546 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -34 15007 528534767 NULL -34.0 15007.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:50.434 1969-12-31 16:00:13.352 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL -50 -13326 528534767 NULL -50.0 -13326.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:46.674 1969-12-31 16:00:08.875 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 16 5780 528534767 NULL 16.0 5780.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.451 1969-12-31 16:00:12.752 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 27 -7824 528534767 NULL 27.0 -7824.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:04.963 1969-12-31 15:59:56.474 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 29 7021 528534767 NULL 29.0 7021.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:15.007 1969-12-31 16:00:15.148 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 31 -9566 528534767 NULL 31.0 -9566.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.187 1969-12-31 16:00:06.961 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 31 4963 528534767 NULL 31.0 4963.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 16:00:07.021 1969-12-31 16:00:02.997 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL 61 -15549 528534767 NULL 61.0 -15549.0 cvLH6Eat2yFsyy7p NULL 1969-12-31 15:59:44.569 1969-12-31 15:59:51.665 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -11534 528534767 NULL NULL -11534.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 15:59:55.407 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -3012 528534767 NULL NULL -3012.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:03.756 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -4016 528534767 NULL NULL -4016.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:07.209 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -4213 528534767 NULL NULL -4213.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.589 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -6147 528534767 NULL NULL -6147.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:13.839 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL -7680 528534767 NULL NULL -7680.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:11.525 true NULL
-NULL -7314 528534767 NULL NULL -7314.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12-31 16:00:06.913 true NULL NULL 11254 528534767 NULL NULL 11254.0 cvLH6Eat2yFsyy7p NULL NULL 1969-12
<TRUNCATED>
[31/61] [abbrv] hive git commit: Revert "HIVE-10685: Alter table
concatenate oparetor will cause duplicate data (guoliming reviewed by
Prasanth Jayachandran)"
Posted by xu...@apache.org.
Revert "HIVE-10685: Alter table concatenate oparetor will cause duplicate data (guoliming reviewed by Prasanth Jayachandran)"
This reverts commit d0c68b29e3b6c57a5d5606c22f938631915274d9.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4f28731c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4f28731c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4f28731c
Branch: refs/heads/spark
Commit: 4f28731cc0da9b2e3a886c49bc59a63f9e332ff8
Parents: d8059d8
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Jun 15 15:22:18 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Jun 15 15:22:18 2015 -0700
----------------------------------------------------------------------
.../hive/ql/io/orc/OrcFileStripeMergeRecordReader.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4f28731c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
index 20c4d31..603b75a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileStripeMergeRecordReader.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.hive.ql.io.orc;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
public class OrcFileStripeMergeRecordReader implements
RecordReader<OrcFileKeyWrapper, OrcFileValueWrapper> {
@@ -79,7 +79,7 @@ public class OrcFileStripeMergeRecordReader implements
// if stripe offset is outside the split boundary then ignore the current
// stripe as it will be handled by some other mapper.
if (si.getOffset() >= start && si.getOffset() < end) {
- valueWrapper.setStripeStatistics(stripeStatistics.get(stripeIdx));
+ valueWrapper.setStripeStatistics(stripeStatistics.get(stripeIdx++));
valueWrapper.setStripeInformation(si);
if (!iter.hasNext()) {
valueWrapper.setLastStripeInFile(true);
@@ -94,7 +94,6 @@ public class OrcFileStripeMergeRecordReader implements
} else {
continue;
}
- stripeIdx++;
return true;
}
[61/61] [abbrv] hive git commit: HIVE-11067: Merge master to Spark
branch 6/20/2015 [Spark Branch]
Posted by xu...@apache.org.
HIVE-11067: Merge master to Spark branch 6/20/2015 [Spark Branch]
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/18990c83
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/18990c83
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/18990c83
Branch: refs/heads/spark
Commit: 18990c8347c0e11ff7408aff5eaae95fe5b3f7b9
Parents: 58fafa2 e09e14a
Author: xzhang <xz...@xzdt>
Authored: Sat Jun 20 22:21:07 2015 -0700
Committer: xzhang <xz...@xzdt>
Committed: Sat Jun 20 22:21:07 2015 -0700
----------------------------------------------------------------------
NOTICE | 2 +-
RELEASE_NOTES.txt | 132 ++
accumulo-handler/pom.xml | 2 +-
.../accumulo/AccumuloConnectionParameters.java | 111 +-
.../hive/accumulo/AccumuloStorageHandler.java | 50 +
.../hive/accumulo/HiveAccumuloHelper.java | 280 ++++
.../mr/HiveAccumuloTableInputFormat.java | 74 +-
.../mr/HiveAccumuloTableOutputFormat.java | 63 +-
.../TestAccumuloConnectionParameters.java | 19 +
.../hive/accumulo/TestHiveAccumuloHelper.java | 75 +
.../mr/TestHiveAccumuloTableInputFormat.java | 8 +-
.../mr/TestHiveAccumuloTableOutputFormat.java | 2 +-
ant/pom.xml | 2 +-
beeline/pom.xml | 2 +-
bin/ext/hiveserver2.cmd | 51 +-
cli/pom.xml | 2 +-
common/pom.xml | 2 +-
.../hadoop/hive/common/JvmPauseMonitor.java | 12 +-
.../hive/common/metrics/LegacyMetrics.java | 51 +-
.../hive/common/metrics/common/Metrics.java | 8 +-
.../common/metrics/common/MetricsFactory.java | 30 +-
.../metrics/metrics2/CodahaleMetrics.java | 41 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +-
.../hive/common/metrics/TestLegacyMetrics.java | 6 +-
.../metrics/metrics2/TestCodahaleMetrics.java | 16 +-
contrib/pom.xml | 2 +-
data/files/sortdp.txt | 32 +
dev-support/jenkins-submit-build.sh | 10 +-
hbase-handler/pom.xml | 2 +-
hcatalog/core/pom.xml | 2 +-
hcatalog/hcatalog-pig-adapter/pom.xml | 2 +-
hcatalog/pom.xml | 2 +-
hcatalog/server-extensions/pom.xml | 2 +-
hcatalog/streaming/pom.xml | 2 +-
hcatalog/webhcat/java-client/pom.xml | 2 +-
hcatalog/webhcat/svr/pom.xml | 2 +-
.../hive/hcatalog/templeton/AppConfig.java | 2 +
.../hive/hcatalog/templeton/HiveDelegator.java | 15 +-
.../hive/hcatalog/templeton/JarDelegator.java | 8 +-
.../hcatalog/templeton/LauncherDelegator.java | 14 +
.../hive/hcatalog/templeton/PigDelegator.java | 13 +-
.../apache/hive/hcatalog/templeton/Server.java | 34 +-
.../hive/hcatalog/templeton/SqoopDelegator.java | 20 +-
.../hcatalog/templeton/StreamingDelegator.java | 3 +-
.../templeton/tool/JobSubmissionConstants.java | 2 +
.../hcatalog/templeton/tool/LaunchMapper.java | 214 ++-
.../templeton/tool/TempletonControllerJob.java | 7 +-
hwi/pom.xml | 2 +-
itests/custom-serde/pom.xml | 2 +-
itests/hcatalog-unit/pom.xml | 2 +-
itests/hive-jmh/pom.xml | 2 +-
itests/hive-minikdc/pom.xml | 22 +-
itests/hive-unit-hadoop2/pom.xml | 2 +-
itests/hive-unit/pom.xml | 2 +-
itests/pom.xml | 3 +-
itests/qtest-accumulo/pom.xml | 508 ++++++
.../test/java/org/apache/hive/TestDummy.java | 31 +
itests/qtest-spark/pom.xml | 2 +-
itests/qtest/pom.xml | 43 +-
.../test/resources/testconfiguration.properties | 4 +-
itests/test-serde/pom.xml | 2 +-
itests/util/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
metastore/pom.xml | 2 +-
.../upgrade/derby/hive-schema-2.0.0.derby.sql | 336 ++++
.../derby/upgrade-1.2.0-to-2.0.0.derby.sql | 3 +
.../scripts/upgrade/derby/upgrade.order.derby | 2 +-
.../upgrade/mssql/hive-schema-2.0.0.mssql.sql | 947 ++++++++++++
.../mssql/upgrade-1.2.0-to-2.0.0.mssql.sql | 5 +
.../scripts/upgrade/mssql/upgrade.order.mssql | 2 +-
.../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 +
.../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 10 +-
.../upgrade/mysql/hive-schema-2.0.0.mysql.sql | 834 ++++++++++
.../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 2 +-
.../mysql/upgrade-1.2.0-to-2.0.0.mysql.sql | 4 +
.../scripts/upgrade/mysql/upgrade.order.mysql | 2 +-
.../upgrade/oracle/hive-schema-2.0.0.oracle.sql | 788 ++++++++++
.../oracle/upgrade-1.2.0-to-2.0.0.oracle.sql | 4 +
.../scripts/upgrade/oracle/upgrade.order.oracle | 2 +-
.../postgres/hive-schema-2.0.0.postgres.sql | 1457 ++++++++++++++++++
.../upgrade-1.2.0-to-2.0.0.postgres.sql | 12 +
.../upgrade/postgres/upgrade.order.postgres | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 111 +-
.../hive/metastore/HiveMetaStoreClient.java | 13 +-
.../hive/metastore/MetaStoreDirectSql.java | 16 +-
.../hive/metastore/MetaStoreSchemaInfo.java | 8 +-
.../hadoop/hive/metastore/ObjectStore.java | 4 +-
.../hive/metastore/StatObjectConverter.java | 16 +-
.../hive/metastore/txn/CompactionInfo.java | 9 +
.../metastore/txn/CompactionTxnHandler.java | 19 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 36 +-
odbc/pom.xml | 2 +-
packaging/pom.xml | 2 +-
pom.xml | 6 +-
ql/pom.xml | 2 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 3 +
.../hive/ql/exec/OrcFileMergeOperator.java | 70 +-
.../hive/ql/exec/PartitionKeySampler.java | 9 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 7 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 2 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 12 +-
.../hive/ql/exec/tez/HashTableLoader.java | 64 +-
.../hive/ql/exec/tez/HiveSplitGenerator.java | 24 +
.../hadoop/hive/ql/io/HiveInputFormat.java | 38 +-
.../hive/ql/io/orc/ColumnStatisticsImpl.java | 217 ++-
.../hive/ql/io/orc/OrcFileKeyWrapper.java | 27 +-
.../io/orc/OrcFileStripeMergeRecordReader.java | 23 +-
.../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 4 +
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 9 +-
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 4 +-
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 2 +-
.../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 15 -
.../hadoop/hive/ql/lockmgr/HiveLockObject.java | 2 +-
.../zookeeper/ZooKeeperHiveLockManager.java | 2 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 11 +-
.../hive/ql/optimizer/ConstantPropagate.java | 12 +-
.../ql/optimizer/ConstantPropagateProcCtx.java | 21 +
.../optimizer/ConstantPropagateProcFactory.java | 69 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 21 +
.../correlation/ReduceSinkDeDuplication.java | 3 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 6 +-
.../hadoop/hive/ql/parse/PTFInvocationSpec.java | 8 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 25 +
.../hadoop/hive/ql/parse/TezCompiler.java | 5 +-
.../hadoop/hive/ql/parse/WindowingSpec.java | 46 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 20 +-
.../hadoop/hive/ql/processors/SetProcessor.java | 3 +
.../hadoop/hive/ql/session/SessionState.java | 22 +
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 3 +-
.../hive/ql/udf/ptf/WindowingTableFunction.java | 56 +-
.../hive/ql/lockmgr/TestDummyTxnManager.java | 119 +-
.../clientnegative/lockneg_try_lock_db_in_use.q | 2 +-
.../clientnegative/ptf_negative_NoWindowDefn.q | 9 +
.../queries/clientpositive/ambiguitycheck.q | 2 -
.../test/queries/clientpositive/cbo_windowing.q | 4 +-
.../clientpositive/dynpart_sort_opt_bucketing.q | 73 +
.../clientpositive/encryption_drop_partition.q | 18 +
.../clientpositive/groupby_grouping_sets6.q | 1 -
.../clientpositive/hybridgrace_hashjoin_2.q | 5 +-
ql/src/test/queries/clientpositive/join42.q | 36 +
ql/src/test/queries/clientpositive/mergejoin.q | 2 +-
ql/src/test/queries/clientpositive/orc_merge9.q | 44 +
.../queries/clientpositive/ql_rewrite_gbtoidx.q | 4 +-
.../queries/clientpositive/selectDistinctStar.q | 2 -
.../test/queries/clientpositive/tez_self_join.q | 32 +
ql/src/test/queries/clientpositive/tez_smb_1.q | 31 +
.../clientpositive/udaf_percentile_approx_23.q | 2 -
ql/src/test/queries/clientpositive/union35.q | 2 -
.../queries/clientpositive/vector_outer_join1.q | 19 +-
.../queries/clientpositive/vector_outer_join2.q | 18 +-
.../queries/clientpositive/vector_outer_join3.q | 17 +-
.../queries/clientpositive/vector_outer_join4.q | 16 +-
.../clientpositive/windowing_windowspec3.q | 20 +
.../lockneg_try_lock_db_in_use.q.out | 9 +-
.../ptf_negative_NoWindowDefn.q.out | 1 +
.../clientpositive/autogen_colalias.q.out | 4 +-
.../results/clientpositive/cbo_windowing.q.out | 24 +-
.../dynpart_sort_opt_bucketing.q.out | 277 ++++
.../encrypted/encryption_drop_partition.q.out | 81 +
.../encrypted/encryption_drop_table.q.out | 2 +-
.../test/results/clientpositive/groupby10.q.out | 140 +-
.../test/results/clientpositive/groupby11.q.out | 70 +-
.../test/results/clientpositive/groupby2.q.out | 33 +-
.../clientpositive/groupby2_map_skew.q.out | 33 +-
.../test/results/clientpositive/groupby8.q.out | 140 +-
.../clientpositive/groupby8_map_skew.q.out | 70 +-
.../results/clientpositive/groupby_cube1.q.out | 29 +-
.../clientpositive/groupby_grouping_sets6.q.out | 46 +-
.../clientpositive/groupby_rollup1.q.out | 29 +-
ql/src/test/results/clientpositive/join42.q.out | 312 ++++
.../results/clientpositive/orc_merge9.q.out | 186 +++
.../clientpositive/spark/groupby10.q.out | 88 +-
.../clientpositive/spark/groupby11.q.out | 44 +-
.../results/clientpositive/spark/groupby2.q.out | 17 +-
.../results/clientpositive/spark/groupby8.q.out | 88 +-
.../spark/groupby8_map_skew.q.out | 44 +-
.../clientpositive/spark/groupby_cube1.q.out | 17 +-
.../clientpositive/spark/groupby_rollup1.q.out | 17 +-
.../clientpositive/tez/cbo_windowing.q.out | 24 +-
.../clientpositive/tez/explainuser_2.q.out | 4 +-
.../results/clientpositive/tez/groupby2.q.out | 17 +-
.../clientpositive/tez/tez_self_join.q.out | 205 +++
.../results/clientpositive/tez/tez_smb_1.q.out | 278 ++++
.../clientpositive/tez/vector_outer_join1.q.out | 252 ++-
.../clientpositive/tez/vector_outer_join2.q.out | 122 +-
.../clientpositive/tez/vector_outer_join3.q.out | 164 +-
.../clientpositive/tez/vector_outer_join4.q.out | 1060 +++++++------
.../udaf_percentile_approx_23.q.out | 10 +-
.../clientpositive/vector_outer_join1.q.out | 244 ++-
.../clientpositive/vector_outer_join2.q.out | 118 +-
.../clientpositive/vector_outer_join3.q.out | 152 +-
.../clientpositive/vector_outer_join4.q.out | 1052 +++++++------
.../clientpositive/windowing_windowspec3.q.out | 80 +
serde/pom.xml | 2 +-
.../hive/serde2/ColumnProjectionUtils.java | 29 +-
service/pom.xml | 2 +-
.../org/apache/hive/service/cli/CLIService.java | 13 -
.../service/cli/operation/SQLOperation.java | 16 +-
.../hive/service/cli/session/HiveSession.java | 12 +-
.../service/cli/session/HiveSessionImpl.java | 28 +-
.../cli/session/HiveSessionImplwithUGI.java | 21 -
.../service/cli/thrift/ThriftCLIService.java | 4 +
.../service/cli/thrift/ThriftHttpServlet.java | 18 +-
.../apache/hive/service/server/HiveServer2.java | 9 +-
shims/0.20S/pom.xml | 2 +-
.../apache/hadoop/mapred/WebHCatJTShim20S.java | 10 +
shims/0.23/pom.xml | 2 +-
.../apache/hadoop/mapred/WebHCatJTShim23.java | 18 +-
shims/aggregator/pom.xml | 2 +-
shims/common/pom.xml | 2 +-
.../apache/hadoop/hive/shims/HadoopShims.java | 5 +
shims/pom.xml | 2 +-
shims/scheduler/pom.xml | 2 +-
spark-client/pom.xml | 4 +-
testutils/pom.xml | 2 +-
215 files changed, 11781 insertions(+), 2129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/18990c83/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/18990c83/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
[60/61] [abbrv] hive git commit: HIVE-10972: DummyTxnManager always
locks the current database in shared mode,
which is incorrect (Aihua Xu via Chaoyu Tang, reviewd by Alan Gates)
Posted by xu...@apache.org.
HIVE-10972: DummyTxnManager always locks the current database in shared mode, which is incorrect (Aihua Xu via Chaoyu Tang, reviewd by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e09e14a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e09e14a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e09e14a4
Branch: refs/heads/spark
Commit: e09e14a484150947a4286276b2855293be6fb2fa
Parents: b97303c
Author: ctang <ct...@gmail.com>
Authored: Sat Jun 20 16:23:00 2015 -0400
Committer: ctang <ct...@gmail.com>
Committed: Sat Jun 20 16:23:00 2015 -0400
----------------------------------------------------------------------
.../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 15 ---
.../hadoop/hive/ql/lockmgr/HiveLockObject.java | 2 +-
.../zookeeper/ZooKeeperHiveLockManager.java | 2 +
.../hive/ql/lockmgr/TestDummyTxnManager.java | 119 ++++++++++++++++++-
.../clientnegative/lockneg_try_lock_db_in_use.q | 2 +-
.../lockneg_try_lock_db_in_use.q.out | 9 +-
6 files changed, 126 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
index ea04415..21ab8ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
@@ -157,21 +157,6 @@ class DummyTxnManager extends HiveTxnManagerImpl {
return;
}
- HiveLockObject.HiveLockObjectData lockData =
- new HiveLockObject.HiveLockObjectData(plan.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "IMPLICIT",
- plan.getQueryStr());
-
- // Lock the database also
- String currentDb = SessionState.get().getCurrentDatabase();
- lockObjects.add(
- new HiveLockObj(
- new HiveLockObject(currentDb, lockData),
- HiveLockMode.SHARED
- )
- );
-
dedupLockObjects(lockObjects);
List<HiveLock> hiveLocks = lockMgr.lock(lockObjects, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index e75a90a..7e93387 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -51,7 +51,7 @@ public class HiveLockObject {
this.queryId = removeDelimiter(queryId);
this.lockTime = removeDelimiter(lockTime);
this.lockMode = removeDelimiter(lockMode);
- this.queryStr = removeDelimiter(queryStr.trim());
+ this.queryStr = removeDelimiter(queryStr == null ? null : queryStr.trim());
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
index 4f86dd9..fb954d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
@@ -255,6 +255,8 @@ public class ZooKeeperHiveLockManager implements HiveLockManager {
private ZooKeeperHiveLock lock (HiveLockObject key, HiveLockMode mode,
boolean keepAlive, boolean parentCreated) throws LockException {
+ LOG.info("Acquiring lock for " + key.getName() + " with mode " + key.getData().getLockMode());
+
int tryNum = 0;
ZooKeeperHiveLock ret = null;
Set<String> conflictingLocks = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
index 5abb729..19f82ad 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDummyTxnManager.java
@@ -18,16 +18,112 @@
package org.apache.hadoop.hive.ql.lockmgr;
-import junit.framework.Assert;
+import static org.mockito.Mockito.*;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLock;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
+@RunWith(MockitoJUnitRunner.class)
public class TestDummyTxnManager {
+ private HiveConf conf = new HiveConf();
+ private HiveTxnManager txnMgr;
+ private Context ctx;
+ private int nextInput = 1;
+
+ @Mock
+ HiveLockManager mockLockManager;
+
+ @Mock
+ QueryPlan mockQueryPlan;
+
+ @Before
+ public void setUp() throws Exception {
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
+ conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, DummyTxnManager.class.getName());
+ SessionState.start(conf);
+ ctx = new Context(conf);
+ LogManager.getRootLogger().setLevel(Level.DEBUG);
+
+ txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ Assert.assertTrue(txnMgr instanceof DummyTxnManager);
+ // Use reflection to set LockManager since creating the object using the
+ // relection in DummyTxnManager won't take Mocked object
+ Field field = DummyTxnManager.class.getDeclaredField("lockMgr");
+ field.setAccessible(true);
+ field.set(txnMgr, mockLockManager);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (txnMgr != null) txnMgr.closeTxnManager();
+ }
+
+ /**
+ * Verifies the current database object is not locked if the table read is against different database
+ * @throws Exception
+ */
+ @Test
+ public void testSingleReadTable() throws Exception {
+ // Setup
+ SessionState.get().setCurrentDatabase("db1");
+
+ List<HiveLock> expectedLocks = new ArrayList<HiveLock>();
+ expectedLocks.add(new ZooKeeperHiveLock("default", new HiveLockObject(), HiveLockMode.SHARED));
+ expectedLocks.add(new ZooKeeperHiveLock("default.table1", new HiveLockObject(), HiveLockMode.SHARED));
+
+ when(mockLockManager.lock(anyListOf(HiveLockObj.class), eq(false))).thenReturn(expectedLocks);
+ doNothing().when(mockLockManager).setContext(any(HiveLockManagerCtx.class));
+ doNothing().when(mockLockManager).close();
+ ArgumentCaptor<List> lockObjsCaptor = ArgumentCaptor.forClass(List.class);
+
+ when(mockQueryPlan.getInputs()).thenReturn(createReadEntities());
+ when(mockQueryPlan.getOutputs()).thenReturn(new HashSet<WriteEntity>());
+
+ // Execute
+ txnMgr.acquireLocks(mockQueryPlan, ctx, "fred");
+
+ // Verify
+ Assert.assertEquals("db1", SessionState.get().getCurrentDatabase());
+ List<HiveLock> resultLocks = ctx.getHiveLocks();
+ Assert.assertEquals(expectedLocks.size(), resultLocks.size());
+ Assert.assertEquals(expectedLocks.get(0).getHiveLockMode(), resultLocks.get(0).getHiveLockMode());
+ Assert.assertEquals(expectedLocks.get(0).getHiveLockObject().getName(), resultLocks.get(0).getHiveLockObject().getName());
+ Assert.assertEquals(expectedLocks.get(1).getHiveLockMode(), resultLocks.get(1).getHiveLockMode());
+ Assert.assertEquals(expectedLocks.get(0).getHiveLockObject().getName(), resultLocks.get(0).getHiveLockObject().getName());
+
+ verify(mockLockManager).lock((List<HiveLockObj>)lockObjsCaptor.capture(), eq(false));
+ List<HiveLockObj> lockObjs = (List<HiveLockObj>)lockObjsCaptor.getValue();
+ Assert.assertEquals(2, lockObjs.size());
+ Assert.assertEquals("default", lockObjs.get(0).getName());
+ Assert.assertEquals(HiveLockMode.SHARED, lockObjs.get(0).mode);
+ Assert.assertEquals("default/table1", lockObjs.get(1).getName());
+ Assert.assertEquals(HiveLockMode.SHARED, lockObjs.get(1).mode);
+ }
@Test
public void testDedupLockObjects() {
@@ -75,4 +171,25 @@ public class TestDummyTxnManager {
Assert.assertEquals(name2, lockObj.getName());
Assert.assertEquals(HiveLockMode.SHARED, lockObj.getMode());
}
+
+ private HashSet<ReadEntity> createReadEntities() {
+ HashSet<ReadEntity> readEntities = new HashSet<ReadEntity>();
+ ReadEntity re = new ReadEntity(newTable(false));
+ readEntities.add(re);
+
+ return readEntities;
+ }
+
+ private Table newTable(boolean isPartitioned) {
+ Table t = new Table("default", "table" + Integer.toString(nextInput++));
+ if (isPartitioned) {
+ FieldSchema fs = new FieldSchema();
+ fs.setName("version");
+ fs.setType("String");
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
+ partCols.add(fs);
+ t.setPartCols(partCols);
+ }
+ return t;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/test/queries/clientnegative/lockneg_try_lock_db_in_use.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/lockneg_try_lock_db_in_use.q b/ql/src/test/queries/clientnegative/lockneg_try_lock_db_in_use.q
index 4127a6f..85bd425 100644
--- a/ql/src/test/queries/clientnegative/lockneg_try_lock_db_in_use.q
+++ b/ql/src/test/queries/clientnegative/lockneg_try_lock_db_in_use.q
@@ -8,7 +8,7 @@ create table tstsrcpart like default.srcpart;
insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11')
select key, value from default.srcpart where ds='2008-04-08' and hr='11';
-lock table tstsrcpart shared;
+lock database lockneg2 shared;
show locks;
lock database lockneg2 exclusive;
http://git-wip-us.apache.org/repos/asf/hive/blob/e09e14a4/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
index 97ab37a..5486151 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
@@ -32,15 +32,14 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Output: lockneg2@tstsrcpart@ds=2008-04-08/hr=11
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: lock table tstsrcpart shared
-PREHOOK: type: LOCKTABLE
-POSTHOOK: query: lock table tstsrcpart shared
-POSTHOOK: type: LOCKTABLE
+PREHOOK: query: lock database lockneg2 shared
+PREHOOK: type: LOCKDATABASE
+POSTHOOK: query: lock database lockneg2 shared
+POSTHOOK: type: LOCKDATABASE
PREHOOK: query: show locks
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks
POSTHOOK: type: SHOWLOCKS
-lockneg2@tstsrcpart SHARED
PREHOOK: query: lock database lockneg2 exclusive
PREHOOK: type: LOCKDATABASE
Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts.
[07/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.8.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.8.out b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.8.out
index 651c4b4..c3454ee 100644
--- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.8.out
+++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.8.out
@@ -120,25 +120,32 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ alias: b
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ Select Operator
+ expressions: key (type: string), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string), _col2 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -146,11 +153,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -160,51 +169,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
+ name: default.srcpart
+ name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-08
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -245,12 +232,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
- ds 2008-04-08
- hr 12
+ ds 2008-04-09
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -291,12 +278,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-09
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -335,14 +322,37 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ /srcpart/ds=2008-04-09/hr=11 [b]
+ /srcpart/ds=2008-04-09/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -350,13 +360,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -366,26 +374,26 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
@@ -393,21 +401,21 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
filter mappings:
- 1 [0, 1]
+ 0 [1, 1]
filter predicates:
- 0
- 1 {(VALUE._col1 = '2008-04-08')}
+ 0 {(VALUE._col1 = '2008-04-08')}
+ 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((((_col0 > 10) and (_col0 < 20)) and (_col5 > 15)) and (_col5 < 25)) (type: boolean)
+ predicate: ((((UDFToDouble(_col3) > 10.0) and (UDFToDouble(_col3) < 20.0)) and (UDFToDouble(_col0) > 15.0)) and (UDFToDouble(_col0) < 25.0)) (type: boolean)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -596,80 +604,25 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: src
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -765,60 +718,39 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
#### A masked pattern was here ####
- Partition
- base file name: hr=11
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 11
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- numFiles 1
- numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
- rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -826,13 +758,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -842,43 +772,43 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Outer Join 0 to 1
+ Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6, _col7
- Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: (((((_col0 > 10) and (_col0 < 20)) and (_col5 > 15)) and (_col5 < 25)) and (_col7 = '2008-04-08')) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 10.0) and (UDFToDouble(_col3) < 20.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -921,8 +851,6 @@ PREHOOK: Input: default@src
PREHOOK: Input: default@srcpart
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: FROM
src a
@@ -936,8 +864,6 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@srcpart
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
17 val_17 17 val_17
17 val_17 17 val_17
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/parallel_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/parallel_join1.q.out b/ql/src/test/results/clientpositive/spark/parallel_join1.q.out
index 15171d9..47555b2 100644
--- a/ql/src/test/results/clientpositive/spark/parallel_join1.q.out
+++ b/ql/src/test/results/clientpositive/spark/parallel_join1.q.out
@@ -38,37 +38,45 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -104,7 +112,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT dest_j1.* FROM dest_j1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/pcr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/pcr.q.out b/ql/src/test/results/clientpositive/spark/pcr.q.out
index 4c22f0b..efadb1d 100644
--- a/ql/src/test/results/clientpositive/spark/pcr.q.out
+++ b/ql/src/test/results/clientpositive/spark/pcr.q.out
@@ -4818,7 +4818,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key = 11) (type: boolean)
+ predicate: (UDFToDouble(key) = 11.0) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), hr (type: string)
@@ -5047,7 +5047,7 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (key = 11) (type: boolean)
+ predicate: (UDFToDouble(key) = 11.0) (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string), ds (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out b/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out
index 78db996..31b25b3 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out
@@ -40,41 +40,56 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((((key > '2') and (key <> '4')) and (key > '1')) and (key > '20')) and (key < '400')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 < '400') (type: boolean)
- Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 < '400') (type: boolean)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col0 > '20') and (((_col1 < 'val_50') or (_col0 > '2')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -83,27 +98,31 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col0 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
+ Filter Operator
+ predicate: ((_col1 > '50') or (_col0 < '50')) (type: boolean)
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
@@ -315,35 +334,34 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((((key > '2') and (key <> '4')) and (key > '1')) and (key > '20')) and (key < '400')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Map 4
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -352,13 +370,13 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+ predicate: ((_col0 > '50') or (_col1 < '50')) (type: boolean)
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string)
+ expressions: _col1 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/ppd_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/ppd_join.q.out b/ql/src/test/results/clientpositive/spark/ppd_join.q.out
index 441fee1..b97431c 100644
--- a/ql/src/test/results/clientpositive/spark/ppd_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/ppd_join.q.out
@@ -37,42 +37,57 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((((key > '2') and (key <> '4')) and (key > '1')) and (key > '20')) and (key < '400')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 < '400') (type: boolean)
- Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ predicate: (_col0 <> '4') (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '1') and ((_col0 > '20') and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 < '400') (type: boolean)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ predicate: ((_col0 > '20') and (((_col1 < 'val_50') or (_col0 > '2')) and (_col0 < '400'))) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 > '2') and (_col0 <> '4')) (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -81,22 +96,26 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col2 (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col3 (type: string)
- outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate: ((_col1 > '50') or (_col0 < '50')) (type: boolean)
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -577,17 +596,17 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((key > '1') and (key < '400')) and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key <> '4')) (type: boolean)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((((key > '2') and (key <> '4')) and (key > '1')) and (key > '20')) and (key < '400')) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Map 3
Map Operator Tree:
@@ -595,18 +614,17 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > '2') and (key < '400')) and (key <> '4')) and (key > '20')) (type: boolean)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((((key > '1') and (key > '20')) and ((value < 'val_50') or (key > '2'))) and (key < '400')) and (key > '2')) and (key <> '4')) and key is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Statistics: Num rows: 2 Data size: 21 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -615,13 +633,13 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+ predicate: ((_col0 > '50') or (_col2 < '50')) (type: boolean)
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col3 (type: string)
+ expressions: _col2 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
File Output Operator
[22/61] [abbrv] hive git commit: HIVE-10977: No need to instantiate
MetaStoreDirectSql when HMS DirectSql is disabled (Chaoyu Tang,
reviewed by Xuefu Zhang and Sergey Shelukhin)
Posted by xu...@apache.org.
HIVE-10977: No need to instantiate MetaStoreDirectSql when HMS DirectSql is disabled (Chaoyu Tang, reviewed by Xuefu Zhang and Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e236314c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e236314c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e236314c
Branch: refs/heads/spark
Commit: e236314c371fbf646fff1f972569e6383cc9e44a
Parents: 671e1a7
Author: ctang <ct...@gmail.com>
Authored: Fri Jun 12 10:18:05 2015 -0400
Committer: ctang <ct...@gmail.com>
Committed: Fri Jun 12 10:20:14 2015 -0400
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/metastore/ObjectStore.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e236314c/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index fd61333..417ecc8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -292,7 +292,9 @@ public class ObjectStore implements RawStore, Configurable {
isInitialized = pm != null;
if (isInitialized) {
expressionProxy = createExpressionProxy(hiveConf);
- directSql = new MetaStoreDirectSql(pm, hiveConf);
+ if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) {
+ directSql = new MetaStoreDirectSql(pm, hiveConf);
+ }
}
LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
" created in the thread with id: " + Thread.currentThread().getId());
[29/61] [abbrv] hive git commit: HIVE-10985 : Value based windowing
on timestamp and double can't handle NULL value (Aihua Xu via Ashutosh
Chauhan)
Posted by xu...@apache.org.
HIVE-10985 : Value based windowing on timestamp and double can't handle NULL value (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b86728de
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b86728de
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b86728de
Branch: refs/heads/spark
Commit: b86728dea0132238d819ad5c7d3855f3aa65157b
Parents: 670f57c
Author: Aihua Xu <ai...@gmail.com>
Authored: Fri Jun 12 07:21:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Jun 15 10:33:01 2015 -0700
----------------------------------------------------------------------
.../hive/ql/udf/ptf/WindowingTableFunction.java | 56 +++++++++-----
.../clientpositive/windowing_windowspec3.q | 20 +++++
.../clientpositive/windowing_windowspec3.q.out | 80 ++++++++++++++++++++
3 files changed, 136 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b86728de/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
index 4635ba2..32471f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
@@ -1161,20 +1161,28 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
@Override
public boolean isGreater(Object v1, Object v2, int amt) {
- long l1 = PrimitiveObjectInspectorUtils.getLong(v1,
- (PrimitiveObjectInspector) expressionDef.getOI());
- long l2 = PrimitiveObjectInspectorUtils.getLong(v2,
- (PrimitiveObjectInspector) expressionDef.getOI());
- return (l1 -l2) > amt;
+ if (v1 != null && v2 != null) {
+ long l1 = PrimitiveObjectInspectorUtils.getLong(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ long l2 = PrimitiveObjectInspectorUtils.getLong(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ return (l1 -l2) > amt;
+ }
+
+ return v1 != null || v2 != null; // True if only one value is null
}
@Override
public boolean isEqual(Object v1, Object v2) {
- long l1 = PrimitiveObjectInspectorUtils.getLong(v1,
- (PrimitiveObjectInspector) expressionDef.getOI());
- long l2 = PrimitiveObjectInspectorUtils.getLong(v2,
- (PrimitiveObjectInspector) expressionDef.getOI());
- return l1 == l2;
+ if (v1 != null && v2 != null) {
+ long l1 = PrimitiveObjectInspectorUtils.getLong(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ long l2 = PrimitiveObjectInspectorUtils.getLong(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ return l1 == l2;
+ }
+
+ return v1 == null && v2 == null; // True if both are null
}
}
@@ -1186,20 +1194,28 @@ public class WindowingTableFunction extends TableFunctionEvaluator {
@Override
public boolean isGreater(Object v1, Object v2, int amt) {
- double d1 = PrimitiveObjectInspectorUtils.getDouble(v1,
- (PrimitiveObjectInspector) expressionDef.getOI());
- double d2 = PrimitiveObjectInspectorUtils.getDouble(v2,
- (PrimitiveObjectInspector) expressionDef.getOI());
- return (d1 -d2) > amt;
+ if (v1 != null && v2 != null) {
+ double d1 = PrimitiveObjectInspectorUtils.getDouble(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ double d2 = PrimitiveObjectInspectorUtils.getDouble(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ return (d1 -d2) > amt;
+ }
+
+ return v1 != null || v2 != null; // True if only one value is null
}
@Override
public boolean isEqual(Object v1, Object v2) {
- double d1 = PrimitiveObjectInspectorUtils.getDouble(v1,
- (PrimitiveObjectInspector) expressionDef.getOI());
- double d2 = PrimitiveObjectInspectorUtils.getDouble(v2,
- (PrimitiveObjectInspector) expressionDef.getOI());
- return d1 == d2;
+ if (v1 != null && v2 != null) {
+ double d1 = PrimitiveObjectInspectorUtils.getDouble(v1,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ double d2 = PrimitiveObjectInspectorUtils.getDouble(v2,
+ (PrimitiveObjectInspector) expressionDef.getOI());
+ return d1 == d2;
+ }
+
+ return v1 == null && v2 == null; // True if both are null
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b86728de/ql/src/test/queries/clientpositive/windowing_windowspec3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec3.q b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
index c87aaff..608a6cf 100644
--- a/ql/src/test/queries/clientpositive/windowing_windowspec3.q
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec3.q
@@ -29,3 +29,23 @@ select deptno, empno, hiredate, salary,
sum(salary) over (partition by deptno order by hiredate range between 10 following and unbounded following),
sum(salary) over (partition by deptno order by hiredate range between unbounded preceding and 10 following)
from emp;
+
+-- Support timestamp datatype. Value in seconds (90days = 90 * 24 * 3600 seconds)
+select deptno, empno, hirets, salary,
+ sum(salary) over (partition by deptno order by hirets range 7776000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 864000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and unbounded following),
+ sum(salary) over (partition by deptno order by hirets range between unbounded preceding and 864000 following)
+from emp;
+
+-- Support double datatype
+select deptno, empno, bonus,
+ avg(bonus) over (partition by deptno order by bonus range 200 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 100 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 100 following and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 following and unbounded following),
+ avg(bonus) over (partition by deptno order by bonus range between unbounded preceding and 200 following)
+from emp;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b86728de/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec3.q.out b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
index bf7797a..42c042f 100644
--- a/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
+++ b/ql/src/test/results/clientpositive/windowing_windowspec3.q.out
@@ -108,3 +108,83 @@ POSTHOOK: Input: default@emp
30 7844 1981-09-08 1500.0 1500.0 3700.0 NULL 2200.0 2200.0 7200.0
30 7654 1981-09-28 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0
30 7900 1981-12-03 950.0 3700.0 3700.0 2750.0 NULL NULL 9400.0
+PREHOOK: query: -- Support timestamp datatype. Value in seconds (90days = 90 * 24 * 3600 seconds)
+select deptno, empno, hirets, salary,
+ sum(salary) over (partition by deptno order by hirets range 7776000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 864000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and unbounded following),
+ sum(salary) over (partition by deptno order by hirets range between unbounded preceding and 864000 following)
+from emp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emp
+#### A masked pattern was here ####
+POSTHOOK: query: -- Support timestamp datatype. Value in seconds (90days = 90 * 24 * 3600 seconds)
+select deptno, empno, hirets, salary,
+ sum(salary) over (partition by deptno order by hirets range 7776000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 7776000 preceding and 864000 preceding),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and 7776000 following),
+ sum(salary) over (partition by deptno order by hirets range between 864000 following and unbounded following),
+ sum(salary) over (partition by deptno order by hirets range between unbounded preceding and 864000 following)
+from emp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emp
+#### A masked pattern was here ####
+10 7988 NULL 1500.0 3000.0 3000.0 NULL NULL 8750.0 3000.0
+10 7987 NULL 1500.0 3000.0 3000.0 NULL NULL 8750.0 3000.0
+10 7782 1981-06-09 00:00:00 2450.0 2450.0 2450.0 NULL NULL 6300.0 5450.0
+10 7839 1981-11-17 00:00:00 5000.0 5000.0 6300.0 NULL 1300.0 1300.0 10450.0
+10 7934 1982-01-23 00:00:00 1300.0 6300.0 6300.0 5000.0 NULL NULL 11750.0
+20 7369 1980-12-17 00:00:00 800.0 800.0 800.0 NULL NULL 10075.0 800.0
+20 7566 1981-04-02 00:00:00 2975.0 2975.0 2975.0 NULL NULL 7100.0 3775.0
+20 7902 1981-12-03 00:00:00 3000.0 3000.0 3000.0 NULL NULL 4100.0 6775.0
+20 7788 1982-12-09 00:00:00 3000.0 3000.0 4100.0 NULL 1100.0 1100.0 9775.0
+20 7876 1983-01-12 00:00:00 1100.0 4100.0 4100.0 3000.0 NULL NULL 10875.0
+30 7499 1981-02-20 00:00:00 1600.0 1600.0 5700.0 NULL 2850.0 6550.0 2850.0
+30 7521 1981-02-22 00:00:00 1250.0 2850.0 5700.0 NULL 2850.0 6550.0 2850.0
+30 7698 1981-05-01 00:00:00 2850.0 5700.0 5700.0 2850.0 NULL 3700.0 5700.0
+30 7844 1981-09-08 00:00:00 1500.0 1500.0 3700.0 NULL 2200.0 2200.0 7200.0
+30 7654 1981-09-28 00:00:00 1250.0 2750.0 3700.0 1500.0 950.0 950.0 8450.0
+30 7900 1981-12-03 00:00:00 950.0 3700.0 3700.0 2750.0 NULL NULL 9400.0
+PREHOOK: query: -- Support double datatype
+select deptno, empno, bonus,
+ avg(bonus) over (partition by deptno order by bonus range 200 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 100 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 100 following and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 following and unbounded following),
+ avg(bonus) over (partition by deptno order by bonus range between unbounded preceding and 200 following)
+from emp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emp
+#### A masked pattern was here ####
+POSTHOOK: query: -- Support double datatype
+select deptno, empno, bonus,
+ avg(bonus) over (partition by deptno order by bonus range 200 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 preceding and 100 preceding),
+ avg(bonus) over (partition by deptno order by bonus range between 100 following and 200 following),
+ avg(bonus) over (partition by deptno order by bonus range between 200 following and unbounded following),
+ avg(bonus) over (partition by deptno order by bonus range between unbounded preceding and 200 following)
+from emp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emp
+#### A masked pattern was here ####
+10 7934 NULL NULL NULL NULL NULL NULL NULL
+10 7987 NULL NULL NULL NULL NULL NULL NULL
+10 7839 NULL NULL NULL NULL NULL NULL NULL
+10 7988 NULL NULL NULL NULL NULL NULL NULL
+10 7782 NULL NULL NULL NULL NULL NULL NULL
+20 7788 NULL NULL NULL NULL NULL NULL NULL
+20 7902 NULL NULL NULL NULL NULL NULL NULL
+20 7876 NULL NULL NULL NULL NULL NULL NULL
+20 7566 NULL NULL NULL NULL NULL NULL NULL
+20 7369 NULL NULL NULL NULL NULL NULL NULL
+30 7698 NULL NULL NULL NULL NULL 550.0 NULL
+30 7900 NULL NULL NULL NULL NULL 550.0 NULL
+30 7844 0.0 0.0 0.0 NULL NULL 733.3333333333334 0.0
+30 7499 300.0 300.0 400.0 NULL 500.0 1400.0 266.6666666666667
+30 7521 500.0 400.0 400.0 300.0 NULL 1400.0 266.6666666666667
+30 7654 1400.0 1400.0 1400.0 NULL NULL NULL 550.0
[55/61] [abbrv] hive git commit: HIVE-11048: Make test cbo_windowing
robust (Ashutosh Chauhan via Gunther Hagleitner)
Posted by xu...@apache.org.
HIVE-11048: Make test cbo_windowing robust (Ashutosh Chauhan via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d7bdadd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d7bdadd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d7bdadd
Branch: refs/heads/spark
Commit: 5d7bdadd698b6b782e4e6fb272c070a49c78f8c3
Parents: e2dd54a
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Fri Jun 19 15:47:47 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Fri Jun 19 15:47:47 2015 -0700
----------------------------------------------------------------------
.../test/queries/clientpositive/cbo_windowing.q | 4 ++--
.../results/clientpositive/cbo_windowing.q.out | 24 ++++++++++----------
.../clientpositive/tez/cbo_windowing.q.out | 24 ++++++++++----------
3 files changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5d7bdadd/ql/src/test/queries/clientpositive/cbo_windowing.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_windowing.q b/ql/src/test/queries/clientpositive/cbo_windowing.q
index 4ccc891..c5bb683 100644
--- a/ql/src/test/queries/clientpositive/cbo_windowing.q
+++ b/ql/src/test/queries/clientpositive/cbo_windowing.q
@@ -8,8 +8,8 @@ set hive.auto.convert.join=false;
-- SORT_QUERY_RESULTS
select count(c_int) over() from cbo_t1;
-select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1;
-select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1;
+select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn;
+select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn) cbo_t1;
select x from (select count(c_int) over() as x, sum(c_float) over() from cbo_t1) cbo_t1;
select 1+sum(c_int) over() from cbo_t1;
select sum(c_int)+sum(sum(c_int)) over() from cbo_t1;
http://git-wip-us.apache.org/repos/asf/hive/blob/5d7bdadd/ql/src/test/results/clientpositive/cbo_windowing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_windowing.q.out b/ql/src/test/results/clientpositive/cbo_windowing.q.out
index 57201b5..bec3f1a 100644
--- a/ql/src/test/results/clientpositive/cbo_windowing.q.out
+++ b/ql/src/test/results/clientpositive/cbo_windowing.q.out
@@ -34,17 +34,17 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18
18
18
-PREHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1
+PREHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn
PREHOOK: type: QUERY
PREHOOK: Input: default@cbo_t1
PREHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-POSTHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1
+POSTHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn
POSTHOOK: type: QUERY
POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-18 18.0 1 1 1 1 1 0.0 1 NULL
+18 18.0 1 1 1 1 1 0.0 NULL NULL
18 18.0 1 1 10 1 1 0.0 1 1.0
18 18.0 1 1 11 1 1 0.0 1 1.0
18 18.0 1 1 12 1 1 0.0 1 1.0
@@ -55,26 +55,26 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18 18.0 1 1 17 1 1 0.0 1 1.0
18 18.0 1 1 18 1 1 0.0 1 1.0
18 18.0 1 1 19 1 1 0.0 1 1.0
-18 18.0 1 1 2 1 1 0.0 1 NULL
+18 18.0 1 1 2 1 1 0.0 NULL NULL
18 18.0 1 1 20 1 1 0.0 1 1.0
-18 18.0 1 1 3 1 1 0.0 1 NULL
-18 18.0 1 1 4 1 1 0.0 1 NULL
+18 18.0 1 1 3 1 1 0.0 1 1.0
+18 18.0 1 1 4 1 1 0.0 1 1.0
18 18.0 1 1 5 1 1 0.0 1 1.0
18 18.0 1 1 6 1 1 0.0 1 1.0
18 18.0 1 1 7 1 1 0.0 1 1.0
18 18.0 1 1 8 1 1 0.0 1 1.0
18 18.0 1 1 9 1 1 0.0 1 1.0
-PREHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1
+PREHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn) cbo_t1
PREHOOK: type: QUERY
PREHOOK: Input: default@cbo_t1
PREHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-POSTHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1
+POSTHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn) cbo_t1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-18 18.0 1 1 1 1 1 0.0 1 NULL
+18 18.0 1 1 1 1 1 0.0 NULL NULL
18 18.0 1 1 10 1 1 0.0 1 1.0
18 18.0 1 1 11 1 1 0.0 1 1.0
18 18.0 1 1 12 1 1 0.0 1 1.0
@@ -85,10 +85,10 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18 18.0 1 1 17 1 1 0.0 1 1.0
18 18.0 1 1 18 1 1 0.0 1 1.0
18 18.0 1 1 19 1 1 0.0 1 1.0
-18 18.0 1 1 2 1 1 0.0 1 NULL
+18 18.0 1 1 2 1 1 0.0 NULL NULL
18 18.0 1 1 20 1 1 0.0 1 1.0
-18 18.0 1 1 3 1 1 0.0 1 NULL
-18 18.0 1 1 4 1 1 0.0 1 NULL
+18 18.0 1 1 3 1 1 0.0 1 1.0
+18 18.0 1 1 4 1 1 0.0 1 1.0
18 18.0 1 1 5 1 1 0.0 1 1.0
18 18.0 1 1 6 1 1 0.0 1 1.0
18 18.0 1 1 7 1 1 0.0 1 1.0
http://git-wip-us.apache.org/repos/asf/hive/blob/5d7bdadd/ql/src/test/results/clientpositive/tez/cbo_windowing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/cbo_windowing.q.out b/ql/src/test/results/clientpositive/tez/cbo_windowing.q.out
index 57201b5..bec3f1a 100644
--- a/ql/src/test/results/clientpositive/tez/cbo_windowing.q.out
+++ b/ql/src/test/results/clientpositive/tez/cbo_windowing.q.out
@@ -34,17 +34,17 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18
18
18
-PREHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1
+PREHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn
PREHOOK: type: QUERY
PREHOOK: Input: default@cbo_t1
PREHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-POSTHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1
+POSTHOOK: query: select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn
POSTHOOK: type: QUERY
POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-18 18.0 1 1 1 1 1 0.0 1 NULL
+18 18.0 1 1 1 1 1 0.0 NULL NULL
18 18.0 1 1 10 1 1 0.0 1 1.0
18 18.0 1 1 11 1 1 0.0 1 1.0
18 18.0 1 1 12 1 1 0.0 1 1.0
@@ -55,26 +55,26 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18 18.0 1 1 17 1 1 0.0 1 1.0
18 18.0 1 1 18 1 1 0.0 1 1.0
18 18.0 1 1 19 1 1 0.0 1 1.0
-18 18.0 1 1 2 1 1 0.0 1 NULL
+18 18.0 1 1 2 1 1 0.0 NULL NULL
18 18.0 1 1 20 1 1 0.0 1 1.0
-18 18.0 1 1 3 1 1 0.0 1 NULL
-18 18.0 1 1 4 1 1 0.0 1 NULL
+18 18.0 1 1 3 1 1 0.0 1 1.0
+18 18.0 1 1 4 1 1 0.0 1 1.0
18 18.0 1 1 5 1 1 0.0 1 1.0
18 18.0 1 1 6 1 1 0.0 1 1.0
18 18.0 1 1 7 1 1 0.0 1 1.0
18 18.0 1 1 8 1 1 0.0 1 1.0
18 18.0 1 1 9 1 1 0.0 1 1.0
-PREHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1
+PREHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn) cbo_t1
PREHOOK: type: QUERY
PREHOOK: Input: default@cbo_t1
PREHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-POSTHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over(), rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(), lag(c_float, 2, c_float) over() from cbo_t1) cbo_t1
+POSTHOOK: query: select * from (select count(c_int) over(), sum(c_float) over(), max(c_int) over(), min(c_int) over(), row_number() over() as rn, rank() over(), dense_rank() over(), percent_rank() over(), lead(c_int, 2, c_int) over(partition by c_boolean order by c_float), lag(c_float, 2, c_float) over(partition by c_boolean order by c_float) from cbo_t1 order by rn) cbo_t1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@cbo_t1
POSTHOOK: Input: default@cbo_t1@dt=2014
#### A masked pattern was here ####
-18 18.0 1 1 1 1 1 0.0 1 NULL
+18 18.0 1 1 1 1 1 0.0 NULL NULL
18 18.0 1 1 10 1 1 0.0 1 1.0
18 18.0 1 1 11 1 1 0.0 1 1.0
18 18.0 1 1 12 1 1 0.0 1 1.0
@@ -85,10 +85,10 @@ POSTHOOK: Input: default@cbo_t1@dt=2014
18 18.0 1 1 17 1 1 0.0 1 1.0
18 18.0 1 1 18 1 1 0.0 1 1.0
18 18.0 1 1 19 1 1 0.0 1 1.0
-18 18.0 1 1 2 1 1 0.0 1 NULL
+18 18.0 1 1 2 1 1 0.0 NULL NULL
18 18.0 1 1 20 1 1 0.0 1 1.0
-18 18.0 1 1 3 1 1 0.0 1 NULL
-18 18.0 1 1 4 1 1 0.0 1 NULL
+18 18.0 1 1 3 1 1 0.0 1 1.0
+18 18.0 1 1 4 1 1 0.0 1 1.0
18 18.0 1 1 5 1 1 0.0 1 1.0
18 18.0 1 1 6 1 1 0.0 1 1.0
18 18.0 1 1 7 1 1 0.0 1 1.0
[51/61] [abbrv] hive git commit: HIVE-7018: Table and Partition
tables have column LINK_TARGET_ID in Mysql scripts but not others (Yongzhi
Chen via Chaoyu Tang)
Posted by xu...@apache.org.
HIVE-7018: Table and Partition tables have column LINK_TARGET_ID in Mysql scripts but not others (Yongzhi Chen via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef6f313c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef6f313c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef6f313c
Branch: refs/heads/spark
Commit: ef6f313c6258c56c0b7e93a0981c6d643db6159a
Parents: 49d8414
Author: ctang <ct...@gmail.com>
Authored: Thu Jun 18 16:17:55 2015 -0400
Committer: ctang <ct...@gmail.com>
Committed: Thu Jun 18 16:17:55 2015 -0400
----------------------------------------------------------------------
.../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 ++++++++++++++++++++
.../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 10 +---
.../upgrade/mysql/hive-schema-2.0.0.mysql.sql | 10 +---
.../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 2 +-
4 files changed, 58 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ef6f313c/metastore/scripts/upgrade/mysql/021-HIVE-7018.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/021-HIVE-7018.mysql.sql b/metastore/scripts/upgrade/mysql/021-HIVE-7018.mysql.sql
new file mode 100644
index 0000000..4f89892
--- /dev/null
+++ b/metastore/scripts/upgrade/mysql/021-HIVE-7018.mysql.sql
@@ -0,0 +1,53 @@
+SELECT '< HIVE-7018 Remove Table and Partition tables column LINK_TARGET_ID from Mysql for other DBs do not have it >' AS ' ';
+
+DELIMITER $$
+DROP PROCEDURE IF EXISTS RM_TLBS_LINKID $$
+DROP PROCEDURE IF EXISTS RM_PARTITIONS_LINKID $$
+DROP PROCEDURE IF EXISTS RM_LINKID $$
+
+/* Call this procedure to drop column LINK_TARGET_ID for TBLS */
+CREATE PROCEDURE RM_TLBS_LINKID()
+ BEGIN
+ IF EXISTS (SELECT * FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_NAME` = 'TBLS' AND `COLUMN_NAME` = 'LINK_TARGET_ID') THEN
+ ALTER TABLE `TBLS`
+ DROP FOREIGN KEY `TBLS_FK3`
+ ;
+ ALTER TABLE `TBLS`
+ DROP KEY `TBLS_N51`
+ ;
+ ALTER TABLE `TBLS`
+ DROP COLUMN `LINK_TARGET_ID`
+ ;
+ END IF;
+ END $$
+
+/* Call this procedure to drop column LINK_TARGET_ID for PARTITIONS */
+CREATE PROCEDURE RM_PARTITIONS_LINKID()
+ BEGIN
+ IF EXISTS (SELECT * FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_NAME` = 'PARTITIONS' AND `COLUMN_NAME` = 'LINK_TARGET_ID') THEN
+ ALTER TABLE `PARTITIONS`
+ DROP FOREIGN KEY `PARTITIONS_FK3`
+ ;
+ ALTER TABLE `PARTITIONS`
+ DROP KEY `PARTITIONS_N51`
+ ;
+ ALTER TABLE `PARTITIONS`
+ DROP COLUMN `LINK_TARGET_ID`
+ ;
+ END IF;
+ END $$
+
+/*
+ * Check and drop column LINK_TARGET_ID
+ */
+CREATE PROCEDURE RM_LINKID()
+ BEGIN
+ call RM_PARTITIONS_LINKID();
+ call RM_TLBS_LINKID();
+ SELECT 'Completed remove LINK_TARGET_ID';
+ END $$
+
+
+DELIMITER ;
+
+CALL RM_LINKID();
http://git-wip-us.apache.org/repos/asf/hive/blob/ef6f313c/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
index bc63f0d..71de138 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
@@ -211,15 +211,12 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`SD_ID` bigint(20) DEFAULT NULL,
`TBL_ID` bigint(20) DEFAULT NULL,
- `LINK_TARGET_ID` bigint(20) DEFAULT NULL,
PRIMARY KEY (`PART_ID`),
UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
KEY `PARTITIONS_N49` (`TBL_ID`),
KEY `PARTITIONS_N50` (`SD_ID`),
- KEY `PARTITIONS_N51` (`LINK_TARGET_ID`),
CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`),
- CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
- CONSTRAINT `PARTITIONS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
+ CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -590,15 +587,12 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
- `LINK_TARGET_ID` bigint(20) DEFAULT NULL,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
KEY `TBLS_N49` (`DB_ID`),
- KEY `TBLS_N51` (`LINK_TARGET_ID`),
CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
- CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`),
- CONSTRAINT `TBLS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `TBLS` (`TBL_ID`)
+ CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
http://git-wip-us.apache.org/repos/asf/hive/blob/ef6f313c/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
index 3a595dd..6547cf1 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
@@ -211,15 +211,12 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` (
`PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`SD_ID` bigint(20) DEFAULT NULL,
`TBL_ID` bigint(20) DEFAULT NULL,
- `LINK_TARGET_ID` bigint(20) DEFAULT NULL,
PRIMARY KEY (`PART_ID`),
UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`),
KEY `PARTITIONS_N49` (`TBL_ID`),
KEY `PARTITIONS_N50` (`SD_ID`),
- KEY `PARTITIONS_N51` (`LINK_TARGET_ID`),
CONSTRAINT `PARTITIONS_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`),
- CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
- CONSTRAINT `PARTITIONS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `PARTITIONS` (`PART_ID`)
+ CONSTRAINT `PARTITIONS_FK2` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
@@ -590,15 +587,12 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`VIEW_EXPANDED_TEXT` mediumtext,
`VIEW_ORIGINAL_TEXT` mediumtext,
- `LINK_TARGET_ID` bigint(20) DEFAULT NULL,
PRIMARY KEY (`TBL_ID`),
UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
KEY `TBLS_N50` (`SD_ID`),
KEY `TBLS_N49` (`DB_ID`),
- KEY `TBLS_N51` (`LINK_TARGET_ID`),
CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
- CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`),
- CONSTRAINT `TBLS_FK3` FOREIGN KEY (`LINK_TARGET_ID`) REFERENCES `TBLS` (`TBL_ID`)
+ CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
/*!40101 SET character_set_client = @saved_cs_client */;
http://git-wip-us.apache.org/repos/asf/hive/blob/ef6f313c/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
index d5a31f7..44a9946 100644
--- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
@@ -1,4 +1,4 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
+SOURCE 021-HIVE-7018.mysql.sql;
UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
-
[08/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out
index 78cd4bc..190485f 100644
--- a/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out
@@ -118,29 +118,36 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ alias: b
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -148,11 +155,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -162,55 +171,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
+ name: default.srcpart
+ name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-08
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -249,14 +232,39 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -264,13 +272,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -280,41 +286,43 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Left Outer Join0 to 1
+ Right Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col5 > 15) and (_col5 < 25)) (type: boolean)
+ predicate: ((UDFToDouble(_col0) > 15.0) and (UDFToDouble(_col0) < 25.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -504,16 +512,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string), _col2 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -714,16 +726,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -785,16 +801,16 @@ STAGE PLANS:
0 {(VALUE._col1 = '2008-04-08')}
1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col7, _col8
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 244 Data size: 2593 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col7 > 15) and (_col7 < 25)) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 15.0) and (UDFToDouble(_col3) < 25.0)) (type: boolean)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col7 (type: string), _col8 (type: string)
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -983,88 +999,25 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: src
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 222 Data size: 2358 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
+ predicate: (((((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) < 20.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1160,60 +1113,39 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) and (UDFToDouble(key) > 15.0)) and (UDFToDouble(key) < 25.0)) and key is not null) (type: boolean)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
#### A masked pattern was here ####
- Partition
- base file name: hr=11
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 11
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- numFiles 1
- numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
- rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -1221,13 +1153,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -1237,66 +1167,62 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Left Outer Join0 to 1
+ Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6, _col7
- Statistics: Num rows: 244 Data size: 2593 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- isSamplingPred: false
- predicate: (((_col5 > 15) and (_col5 < 25)) and (_col7 = '2008-04-08')) (type: boolean)
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
- Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- columns _col0,_col1,_col2,_col3
- columns.types string:string:string:string
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3
+ columns.types string:string:string:string
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
Stage: Stage-0
Fetch Operator
@@ -1316,8 +1242,6 @@ PREHOOK: Input: default@src
PREHOOK: Input: default@srcpart
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: FROM
src a
@@ -1331,8 +1255,6 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@srcpart
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
17 val_17 17 val_17
17 val_17 17 val_17
@@ -1467,16 +1389,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1583,16 +1509,20 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((key > 10) and (key < 20)) (type: boolean)
+ predicate: ((UDFToDouble(key) > 10.0) and (UDFToDouble(key) < 20.0)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string)
- auto parallelism: false
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -1649,16 +1579,16 @@ STAGE PLANS:
condition map:
Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col7, _col8
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((_col7 > 15) and (_col7 < 25)) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 15.0) and (UDFToDouble(_col3) < 25.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col7 (type: string), _col8 (type: string)
+ expressions: _col0 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/merge1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/merge1.q.out b/ql/src/test/results/clientpositive/spark/merge1.q.out
index 41acbdb..e594d47 100644
--- a/ql/src/test/results/clientpositive/spark/merge1.q.out
+++ b/ql/src/test/results/clientpositive/spark/merge1.q.out
@@ -42,11 +42,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/merge2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/merge2.q.out b/ql/src/test/results/clientpositive/spark/merge2.q.out
index edeebb7..63274eb 100644
--- a/ql/src/test/results/clientpositive/spark/merge2.q.out
+++ b/ql/src/test/results/clientpositive/spark/merge2.q.out
@@ -42,11 +42,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
- outputColumnNames: key
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out b/ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out
index 0ccae50..18e1d4e 100644
--- a/ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out
+++ b/ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out
@@ -200,10 +200,10 @@ STAGE PLANS:
Statistics: Num rows: 9999 Data size: 1030908 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s (type: string), bo (type: boolean), bin (type: binary), si (type: smallint), i (type: int), b (type: bigint)
- outputColumnNames: s, bo, bin, si, i, b
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 9999 Data size: 1030908 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si), max(i), min(b)
+ aggregations: count(), sum(1), sum(0.2), count(1), count(_col2), count(_col3), count(_col4), count(_col5), max(_col6), min(_col7)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
@@ -256,10 +256,10 @@ STAGE PLANS:
Statistics: Num rows: 9489 Data size: 978785 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s (type: string), bo (type: boolean), bin (type: binary), si (type: smallint), i (type: int), b (type: bigint)
- outputColumnNames: s, bo, bin, si, i, b
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 9489 Data size: 978785 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si), max(i), min(b)
+ aggregations: count(), sum(1), sum(0.2), count(1), count(_col2), count(_col3), count(_col4), count(_col5), max(_col6), min(_col7)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE
@@ -448,10 +448,10 @@ STAGE PLANS:
Statistics: Num rows: 9489 Data size: 978785 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ts (type: timestamp)
- outputColumnNames: ts
+ outputColumnNames: _col0
Statistics: Num rows: 9489 Data size: 978785 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(ts)
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/order2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/order2.q.out b/ql/src/test/results/clientpositive/spark/order2.q.out
index 22b1ddc..29eb835 100644
--- a/ql/src/test/results/clientpositive/spark/order2.q.out
+++ b/ql/src/test/results/clientpositive/spark/order2.q.out
@@ -43,7 +43,7 @@ STAGE PLANS:
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col0 < 10) (type: boolean)
+ predicate: (UDFToDouble(_col0) < 10.0) (type: boolean)
Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
index 227886b..0614cb9 100644
--- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
@@ -120,25 +120,32 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ alias: b
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
+ Select Operator
+ expressions: key (type: string), value (type: string), ds (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string), _col2 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: src
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -146,11 +153,13 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
+ name default.srcpart
numFiles 1
numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -160,51 +169,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
+ name default.srcpart
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct srcpart { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
+ name: default.srcpart
+ name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-08
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -245,12 +232,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: hr=11
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
- ds 2008-04-08
- hr 12
+ ds 2008-04-09
+ hr 11
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -291,12 +278,12 @@ STAGE PLANS:
name: default.srcpart
#### A masked pattern was here ####
Partition
- base file name: hr=11
+ base file name: hr=12
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
partition values:
ds 2008-04-09
- hr 11
+ hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -335,14 +322,37 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ /srcpart/ds=2008-04-09/hr=11 [b]
+ /srcpart/ds=2008-04-09/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -350,13 +360,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -366,26 +374,26 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
@@ -393,21 +401,21 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
filter mappings:
- 1 [0, 1]
+ 0 [1, 1]
filter predicates:
- 0
- 1 {(VALUE._col1 = '2008-04-08')}
+ 0 {(VALUE._col1 = '2008-04-08')}
+ 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: ((((_col5 > 15) and (_col5 < 25)) and (_col0 > 10)) and (_col0 < 20)) (type: boolean)
+ predicate: ((((UDFToDouble(_col0) > 15.0) and (UDFToDouble(_col0) < 25.0)) and (UDFToDouble(_col3) > 10.0)) and (UDFToDouble(_col3) < 20.0)) (type: boolean)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -596,80 +604,25 @@ STAGE PLANS:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- value expressions: value (type: string)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: src
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src
- name: default.src
- Truncated Path -> Alias:
- /src [a]
- Map 3
- Map Operator Tree:
- TableScan
alias: b
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- value expressions: value (type: string), ds (type: string)
- auto parallelism: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -765,60 +718,39 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcpart
name: default.srcpart
+ Truncated Path -> Alias:
+ /srcpart/ds=2008-04-08/hr=11 [b]
+ /srcpart/ds=2008-04-08/hr=12 [b]
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((UDFToDouble(key) > 15.0) and (UDFToDouble(key) < 25.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ tag: 1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
#### A masked pattern was here ####
- Partition
- base file name: hr=11
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 11
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- numFiles 1
- numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
- rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
-#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ Path -> Partition:
#### A masked pattern was here ####
Partition
- base file name: hr=12
+ base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- hr 12
properties:
COLUMN_STATS_ACCURATE true
bucket_count -1
@@ -826,13 +758,11 @@ STAGE PLANS:
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
+ name default.src
numFiles 1
numRows 500
- partition_columns ds/hr
- partition_columns.types string:string
rawDataSize 5312
- serialization.ddl struct srcpart { string key, string value}
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -842,43 +772,43 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ COLUMN_STATS_ACCURATE true
bucket_count -1
columns key,value
columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
- name default.srcpart
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct srcpart { string key, string value}
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcpart
- name: default.srcpart
+ name: default.src
+ name: default.src
Truncated Path -> Alias:
- /srcpart/ds=2008-04-08/hr=11 [b]
- /srcpart/ds=2008-04-08/hr=12 [b]
- /srcpart/ds=2008-04-09/hr=11 [b]
- /srcpart/ds=2008-04-09/hr=12 [b]
+ /src [a]
Reducer 2
Needs Tagging: true
Reduce Operator Tree:
Join Operator
condition map:
- Outer Join 0 to 1
+ Left Outer Join0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6, _col7
- Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 122 Data size: 1296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
isSamplingPred: false
- predicate: (((((_col5 > 15) and (_col5 < 25)) and (_col7 = '2008-04-08')) and (_col0 > 10)) and (_col0 < 20)) (type: boolean)
+ predicate: ((UDFToDouble(_col3) > 10.0) and (UDFToDouble(_col3) < 20.0)) (type: boolean)
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ expressions: _col3 (type: string), _col4 (type: string), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -921,8 +851,6 @@ PREHOOK: Input: default@src
PREHOOK: Input: default@srcpart
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
POSTHOOK: query: FROM
src a
@@ -936,8 +864,6 @@ POSTHOOK: Input: default@src
POSTHOOK: Input: default@srcpart
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
17 val_17 17 val_17
17 val_17 17 val_17
[12/61] [abbrv] hive git commit: HIVE-10903: Add hive.in.test for HoS
tests (Rui reviewed by Xuefu)
Posted by xu...@apache.org.
HIVE-10903: Add hive.in.test for HoS tests (Rui reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/330c31ae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/330c31ae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/330c31ae
Branch: refs/heads/spark
Commit: 330c31ae637daca5c7085b1610b7adcb65af297a
Parents: 4757302
Author: Rui Li <ru...@intel.com>
Authored: Wed Jun 10 15:05:47 2015 +0800
Committer: Rui Li <ru...@intel.com>
Committed: Wed Jun 10 15:06:30 2015 +0800
----------------------------------------------------------------------
data/conf/spark/standalone/hive-site.xml | 6 +
data/conf/spark/yarn-client/hive-site.xml | 6 +
.../test/queries/clientpositive/cbo_subq_in.q | 1 +
...groupby_complex_types_multi_single_reducer.q | 8 +-
.../results/clientpositive/cbo_subq_in.q.out | 2 +
...pby_complex_types_multi_single_reducer.q.out | 68 +--
.../spark/annotate_stats_join.q.out | 204 +++----
.../clientpositive/spark/auto_join1.q.out | 60 +-
.../clientpositive/spark/auto_join10.q.out | 28 +-
.../clientpositive/spark/auto_join11.q.out | 32 +-
.../clientpositive/spark/auto_join14.q.out | 64 +-
.../clientpositive/spark/auto_join16.q.out | 68 ++-
.../clientpositive/spark/auto_join17.q.out | 60 +-
.../clientpositive/spark/auto_join18.q.out | 30 +-
.../spark/auto_join18_multi_distinct.q.out | 30 +-
.../clientpositive/spark/auto_join19.q.out | 56 +-
.../clientpositive/spark/auto_join26.q.out | 68 ++-
.../clientpositive/spark/auto_join27.q.out | 24 +-
.../clientpositive/spark/auto_join4.q.out | 14 +-
.../clientpositive/spark/auto_join5.q.out | 24 +-
.../clientpositive/spark/auto_join6.q.out | 10 +-
.../clientpositive/spark/auto_join8.q.out | 16 +-
.../clientpositive/spark/auto_join9.q.out | 56 +-
.../spark/auto_join_filters.q.out | 4 +-
.../clientpositive/spark/auto_join_nulls.q.out | 6 +-
.../spark/auto_join_without_localtask.q.out | 32 +-
.../clientpositive/spark/cbo_subq_in.q.out | 2 +
.../spark/constprog_partitioner.q.out | 80 +--
.../results/clientpositive/spark/count.q.out | 14 +-
.../clientpositive/spark/cross_join.q.out | 74 ++-
.../spark/cross_product_check_1.q.out | 40 +-
.../spark/cross_product_check_2.q.out | 38 +-
.../results/clientpositive/spark/groupby1.q.out | 8 +-
.../results/clientpositive/spark/groupby4.q.out | 6 +-
...pby_complex_types_multi_single_reducer.q.out | 90 ++-
.../clientpositive/spark/groupby_cube1.q.out | 24 +-
.../clientpositive/spark/groupby_position.q.out | 130 +++--
.../clientpositive/spark/groupby_rollup1.q.out | 20 +-
.../spark/groupby_sort_1_23.q.out | 82 +--
.../spark/groupby_sort_skew_1_23.q.out | 82 +--
.../results/clientpositive/spark/having.q.out | 66 ++-
.../spark/index_auto_self_join.q.out | 116 ++--
.../spark/infer_bucket_sort_map_operators.q.out | 5 +-
.../spark/infer_bucket_sort_merge.q.out | 4 +-
.../clientpositive/spark/innerjoin.q.out | 54 +-
.../results/clientpositive/spark/join1.q.out | 38 +-
.../results/clientpositive/spark/join10.q.out | 28 +-
.../results/clientpositive/spark/join11.q.out | 18 +-
.../results/clientpositive/spark/join14.q.out | 54 +-
.../results/clientpositive/spark/join16.q.out | 36 +-
.../results/clientpositive/spark/join17.q.out | 52 +-
.../results/clientpositive/spark/join18.q.out | 12 +-
.../spark/join18_multi_distinct.q.out | 12 +-
.../results/clientpositive/spark/join29.q.out | 66 ++-
.../results/clientpositive/spark/join31.q.out | 104 ++--
.../results/clientpositive/spark/join34.q.out | 36 +-
.../results/clientpositive/spark/join35.q.out | 92 +--
.../results/clientpositive/spark/join4.q.out | 16 +-
.../results/clientpositive/spark/join40.q.out | 116 ++--
.../results/clientpositive/spark/join5.q.out | 20 +-
.../results/clientpositive/spark/join6.q.out | 10 +-
.../results/clientpositive/spark/join8.q.out | 18 +-
.../results/clientpositive/spark/join9.q.out | 42 +-
.../clientpositive/spark/join_alt_syntax.q.out | 42 +-
.../results/clientpositive/spark/join_vc.q.out | 40 +-
.../spark/limit_partition_metadataonly.q.out | 4 +-
.../clientpositive/spark/limit_pushdown.q.out | 138 ++---
.../clientpositive/spark/load_dyn_part13.q.out | 4 +-
.../clientpositive/spark/louter_join_ppr.q.out | 514 +++++++---------
.../results/clientpositive/spark/merge1.q.out | 4 +-
.../results/clientpositive/spark/merge2.q.out | 4 +-
.../spark/metadata_only_queries.q.out | 12 +-
.../results/clientpositive/spark/order2.q.out | 2 +-
.../spark/outer_join_ppr.q.java1.7.out | 366 +++++-------
.../spark/outer_join_ppr.q.java1.8.out | 366 +++++-------
.../clientpositive/spark/parallel_join1.q.out | 38 +-
.../test/results/clientpositive/spark/pcr.q.out | 4 +-
.../clientpositive/spark/ppd_gby_join.q.out | 132 +++--
.../results/clientpositive/spark/ppd_join.q.out | 118 ++--
.../clientpositive/spark/ppd_join_filter.q.out | 442 +++++++-------
.../clientpositive/spark/ppd_outer_join1.q.out | 120 ++--
.../clientpositive/spark/ppd_outer_join2.q.out | 158 ++---
.../clientpositive/spark/ppd_outer_join3.q.out | 154 +++--
.../spark/ql_rewrite_gbtoidx_cbo_1.q.out | 208 +++----
.../clientpositive/spark/router_join_ppr.q.out | 582 ++++++++-----------
.../results/clientpositive/spark/semijoin.q.out | 26 +-
.../results/clientpositive/spark/skewjoin.q.out | 160 ++---
.../clientpositive/spark/skewjoin_noskew.q.out | 32 +-
.../results/clientpositive/spark/stats1.q.out | 2 +-
.../clientpositive/spark/stats_only_null.q.out | 8 +-
.../clientpositive/spark/subquery_exists.q.out | 20 +-
.../clientpositive/spark/subquery_in.q.out | 230 ++++----
.../spark/subquery_multiinsert.q.java1.8.out | 14 +-
.../spark/table_access_keys_stats.q.out | 18 +-
.../clientpositive/spark/temp_table.q.out | 4 +-
.../results/clientpositive/spark/union.q.out | 4 +-
.../results/clientpositive/spark/union10.q.out | 6 +-
.../results/clientpositive/spark/union11.q.out | 82 ++-
.../results/clientpositive/spark/union12.q.out | 2 +-
.../results/clientpositive/spark/union13.q.out | 2 +-
.../results/clientpositive/spark/union14.q.out | 52 +-
.../results/clientpositive/spark/union15.q.out | 80 ++-
.../results/clientpositive/spark/union2.q.out | 2 +-
.../results/clientpositive/spark/union20.q.out | 10 +-
.../results/clientpositive/spark/union24.q.out | 36 +-
.../results/clientpositive/spark/union25.q.out | 38 +-
.../results/clientpositive/spark/union28.q.out | 78 +--
.../results/clientpositive/spark/union30.q.out | 78 +--
.../results/clientpositive/spark/union33.q.out | 12 +-
.../results/clientpositive/spark/union4.q.out | 4 +-
.../results/clientpositive/spark/union5.q.out | 54 +-
.../results/clientpositive/spark/union6.q.out | 2 +-
.../results/clientpositive/spark/union7.q.out | 52 +-
.../results/clientpositive/spark/union8.q.out | 4 +-
.../results/clientpositive/spark/union9.q.out | 4 +-
.../clientpositive/spark/union_remove_1.q.out | 8 +-
.../clientpositive/spark/union_remove_10.q.out | 8 +-
.../clientpositive/spark/union_remove_15.q.out | 8 +-
.../clientpositive/spark/union_remove_16.q.out | 8 +-
.../clientpositive/spark/union_remove_18.q.out | 8 +-
.../clientpositive/spark/union_remove_19.q.out | 192 +++---
.../clientpositive/spark/union_remove_2.q.out | 8 +-
.../clientpositive/spark/union_remove_20.q.out | 8 +-
.../clientpositive/spark/union_remove_21.q.out | 70 +--
.../clientpositive/spark/union_remove_22.q.out | 8 +-
.../clientpositive/spark/union_remove_24.q.out | 8 +-
.../clientpositive/spark/union_remove_25.q.out | 8 +-
.../clientpositive/spark/union_remove_4.q.out | 8 +-
.../clientpositive/spark/union_remove_5.q.out | 8 +-
.../spark/union_remove_6_subq.q.out | 8 +-
.../clientpositive/spark/union_remove_7.q.out | 8 +-
.../clientpositive/spark/union_remove_8.q.out | 8 +-
.../clientpositive/spark/union_remove_9.q.out | 4 +-
.../clientpositive/spark/union_top_level.q.out | 24 +-
.../spark/vector_cast_constant.q.java1.7.out | 6 +-
.../spark/vector_cast_constant.q.java1.8.out | 6 +-
.../spark/vector_decimal_aggregate.q.out | 12 +-
.../spark/vector_distinct_2.q.out | 4 +-
.../clientpositive/spark/vector_elt.q.out | 2 +-
.../clientpositive/spark/vector_groupby_3.q.out | 6 +-
.../spark/vector_mapjoin_reduce.q.out | 261 +++++----
.../clientpositive/spark/vector_orderby_5.q.out | 6 +-
.../spark/vector_string_concat.q.out | 6 +-
.../clientpositive/spark/vectorization_0.q.out | 20 +-
.../clientpositive/spark/vectorization_13.q.out | 20 +-
.../clientpositive/spark/vectorization_14.q.out | 12 +-
.../clientpositive/spark/vectorization_15.q.out | 10 +-
.../clientpositive/spark/vectorization_16.q.out | 8 +-
.../clientpositive/spark/vectorization_9.q.out | 8 +-
.../spark/vectorization_div0.q.out | 6 +-
.../spark/vectorization_part_project.q.out | 2 +-
.../spark/vectorization_pushdown.q.out | 8 +-
.../spark/vectorization_short_regress.q.out | 112 ++--
.../clientpositive/spark/vectorized_case.q.out | 2 +-
.../spark/vectorized_mapjoin.q.out | 60 +-
.../spark/vectorized_shufflejoin.q.out | 52 +-
.../spark/vectorized_timestamp_funcs.q.out | 12 +-
.../clientpositive/tez/cbo_subq_in.q.out | 2 +
158 files changed, 4207 insertions(+), 4096 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/data/conf/spark/standalone/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml
index 016f568..abc73f8 100644
--- a/data/conf/spark/standalone/hive-site.xml
+++ b/data/conf/spark/standalone/hive-site.xml
@@ -230,4 +230,10 @@
<value>hive_admin_user</value>
</property>
+<property>
+ <name>hive.in.test</name>
+ <value>true</value>
+ <description>Internal marker for test. Used for masking env-dependent values</description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/data/conf/spark/yarn-client/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml
index 39ba20e..b7211ee 100644
--- a/data/conf/spark/yarn-client/hive-site.xml
+++ b/data/conf/spark/yarn-client/hive-site.xml
@@ -250,4 +250,10 @@
<value>hive_admin_user</value>
</property>
+<property>
+ <name>hive.in.test</name>
+ <value>true</value>
+ <description>Internal marker for test. Used for masking env-dependent values</description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/queries/clientpositive/cbo_subq_in.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_subq_in.q b/ql/src/test/queries/clientpositive/cbo_subq_in.q
index 7a0bfba..0a25b9c 100644
--- a/ql/src/test/queries/clientpositive/cbo_subq_in.q
+++ b/ql/src/test/queries/clientpositive/cbo_subq_in.q
@@ -29,6 +29,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
;
-- where and having
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q b/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q
index b7e1bf1..0a1f137 100644
--- a/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q
+++ b/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q
@@ -7,12 +7,12 @@ CREATE TABLE DEST2(key MAP<STRING, STRING>, value BIGINT) STORED AS TEXTFILE;
EXPLAIN
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10;
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10;
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10;
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10;
SELECT DEST1.* FROM DEST1;
SELECT DEST2.* FROM DEST2;
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/cbo_subq_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_subq_in.q.out b/ql/src/test/results/clientpositive/cbo_subq_in.q.out
index c1f3de7..f6bfad2 100644
--- a/ql/src/test/results/clientpositive/cbo_subq_in.q.out
+++ b/ql/src/test/results/clientpositive/cbo_subq_in.q.out
@@ -69,6 +69,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
PREHOOK: type: QUERY
PREHOOK: Input: default@lineitem
#### A masked pattern was here ####
@@ -77,6 +78,7 @@ select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
where li.l_linenumber = 1 and
li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber)
+ order by p.p_partkey
POSTHOOK: type: QUERY
POSTHOOK: Input: default@lineitem
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out b/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
index 4611f60..0564056 100644
--- a/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
+++ b/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
@@ -20,13 +20,13 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@DEST2
PREHOOK: query: EXPLAIN
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
@@ -84,29 +84,27 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-3
Map Reduce
Map Operator Tree:
TableScan
Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: array<string>), _col1 (type: bigint)
+ key expressions: _col0 (type: array<string>)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: array<string>), VALUE._col1 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: array<string>), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -149,29 +147,27 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-6
Map Reduce
Map Operator Tree:
TableScan
Reduce Output Operator
- sort order:
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: map<string,string>), _col1 (type: bigint)
+ key expressions: _col0 (type: map<string,string>)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: map<string,string>), VALUE._col1 (type: bigint)
+ expressions: KEY.reducesinkkey0 (type: map<string,string>), VALUE._col0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
@@ -198,15 +194,15 @@ STAGE PLANS:
Stats-Aggr Operator
PREHOOK: query: FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@dest1
PREHOOK: Output: default@dest2
POSTHOOK: query: FROM SRC
-INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) limit 10
-INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value), COUNT(1) GROUP BY MAP(SRC.key, SRC.value) limit 10
+INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10
+INSERT OVERWRITE TABLE DEST2 SELECT MAP(SRC.key, SRC.value) as kvmap, COUNT(1) GROUP BY MAP(SRC.key, SRC.value) ORDER BY kvmap limit 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
index 032926d..8955a61 100644
--- a/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
+++ b/ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out
@@ -173,12 +173,16 @@ STAGE PLANS:
Filter Operator
predicate: deptid is not null (type: boolean)
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int)
- sort order: +
- Map-reduce partition columns: deptid (type: int)
+ Select Operator
+ expressions: lastname (type: string), deptid (type: int), locid (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: lastname (type: string), locid (type: int)
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -187,33 +191,33 @@ STAGE PLANS:
Filter Operator
predicate: deptid is not null (type: boolean)
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int)
- sort order: +
- Map-reduce partition columns: deptid (type: int)
+ Select Operator
+ expressions: deptid (type: int), deptname (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: deptname (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 deptid (type: int)
- 1 deptid (type: int)
- outputColumnNames: _col0, _col1, _col2, _col6, _col7
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 48 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ File Output Operator
+ compressed: false
Statistics: Num rows: 48 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 48 Data size: 9312 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -248,12 +252,16 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and lastname is not null) (type: boolean)
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), lastname (type: string)
- sort order: ++
- Map-reduce partition columns: deptid (type: int), lastname (type: string)
+ Select Operator
+ expressions: lastname (type: string), deptid (type: int), locid (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: locid (type: int)
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: int), _col0 (type: string)
+ Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -262,35 +270,32 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and deptname is not null) (type: boolean)
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), deptname (type: string)
- sort order: ++
- Map-reduce partition columns: deptid (type: int), deptname (type: string)
+ Select Operator
+ expressions: deptid (type: int), deptname (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 deptid (type: int), lastname (type: string)
- 1 deptid (type: int), deptname (type: string)
- outputColumnNames: _col0, _col1, _col2, _col6, _col7
+ 0 _col1 (type: int), _col0 (type: string)
+ 1 _col0 (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 6 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((_col1 = _col6) and (_col0 = _col7)) (type: boolean)
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -321,12 +326,16 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and lastname is not null) (type: boolean)
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), lastname (type: string)
- sort order: ++
- Map-reduce partition columns: deptid (type: int), lastname (type: string)
+ Select Operator
+ expressions: lastname (type: string), deptid (type: int), locid (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: locid (type: int)
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: int), _col0 (type: string)
+ Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -335,32 +344,32 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and deptname is not null) (type: boolean)
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), deptname (type: string)
- sort order: ++
- Map-reduce partition columns: deptid (type: int), deptname (type: string)
+ Select Operator
+ expressions: deptid (type: int), deptname (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string)
+ Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 deptid (type: int), lastname (type: string)
- 1 deptid (type: int), deptname (type: string)
- outputColumnNames: _col0, _col1, _col2, _col6, _col7
+ 0 _col1 (type: int), _col0 (type: string)
+ 1 _col0 (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 6 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ File Output Operator
+ compressed: false
Statistics: Num rows: 6 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 1164 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -395,12 +404,16 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and lastname is not null) (type: boolean)
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), lastname (type: string), lastname (type: string)
- sort order: +++
- Map-reduce partition columns: deptid (type: int), lastname (type: string), lastname (type: string)
+ Select Operator
+ expressions: lastname (type: string), deptid (type: int), locid (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: locid (type: int)
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col0 (type: string), _col0 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col0 (type: string)
+ Statistics: Num rows: 48 Data size: 4752 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: int)
Map 3
Map Operator Tree:
TableScan
@@ -409,35 +422,32 @@ STAGE PLANS:
Filter Operator
predicate: (deptid is not null and deptname is not null) (type: boolean)
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: deptid (type: int), deptname (type: string), deptname (type: string)
- sort order: +++
- Map-reduce partition columns: deptid (type: int), deptname (type: string), deptname (type: string)
+ Select Operator
+ expressions: deptid (type: int), deptname (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string), _col1 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col1 (type: string)
+ Statistics: Num rows: 6 Data size: 570 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 deptid (type: int), lastname (type: string), lastname (type: string)
- 1 deptid (type: int), deptname (type: string), deptname (type: string)
- outputColumnNames: _col0, _col1, _col2, _col6, _col7
+ 0 _col1 (type: int), _col0 (type: string), _col0 (type: string)
+ 1 _col0 (type: int), _col1 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 11 Data size: 2134 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (((_col1 = _col6) and (_col0 = _col7)) and (_col7 = _col0)) (type: boolean)
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col6 (type: int), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 194 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 2134 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join1.q.out b/ql/src/test/results/clientpositive/spark/auto_join1.q.out
index d9215f8..d26a33e 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join1.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join1.q.out
@@ -32,15 +32,19 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
@@ -56,28 +60,32 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
- input vertices:
- 1 Map 2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest_j1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest_j1
Local Work:
Map Reduce Local Work
@@ -105,7 +113,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest_j1
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest_j1.value SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest_j1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join10.q.out b/ql/src/test/results/clientpositive/spark/auto_join10.q.out
index cd50576..b43e55c 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join10.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join10.q.out
@@ -33,8 +33,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
@@ -58,8 +58,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -67,19 +67,23 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col2, _col3
+ outputColumnNames: _col0, _col1
input vertices:
1 Map 3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col2,_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col0,_col1) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
Reducer 2
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join11.q.out b/ql/src/test/results/clientpositive/spark/auto_join11.q.out
index ed32dec..f8fc309 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join11.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join11.q.out
@@ -30,11 +30,11 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
@@ -55,11 +55,11 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -67,19 +67,23 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col3
+ outputColumnNames: _col1, _col2
input vertices:
1 Map 3
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col0,_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col2,_col1) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
Reducer 2
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join14.q.out b/ql/src/test/results/clientpositive/spark/auto_join14.q.out
index 830314e..30be2e8 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join14.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join14.q.out
@@ -29,18 +29,22 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 2
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key > 100) (type: boolean)
+ predicate: (UDFToDouble(key) > 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
@@ -48,36 +52,40 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 1
Map Operator Tree:
TableScan
alias: srcpart
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((key > 100) and key is not null) (type: boolean)
+ predicate: ((UDFToDouble(key) > 100.0) and key is not null) (type: boolean)
Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col6
- input vertices:
- 0 Map 1
- Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 167 Data size: 1774 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col3
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: UDFToInteger(_col3) (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 183 Data size: 1951 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join16.q.out b/ql/src/test/results/clientpositive/spark/auto_join16.q.out
index 8c166b0..5c4bbb3 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join16.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join16.q.out
@@ -24,61 +24,69 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > 10) and value is not null) and (key > 20)) and (value < 200)) (type: boolean)
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((((UDFToDouble(value) < 200.0) and (UDFToDouble(key) > 10.0)) and (UDFToDouble(key) > 20.0)) and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 9 Data size: 95 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: string), _col1 (type: string)
- 1 key (type: string), value (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 1
Map Operator Tree:
TableScan
- alias: tab
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((key > 20) and value is not null) and key is not null) and (value < 200)) (type: boolean)
- Statistics: Num rows: 14 Data size: 148 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string), _col1 (type: string)
- 1 key (type: string), value (type: string)
- outputColumnNames: _col0, _col3
- input vertices:
- 0 Map 1
- Statistics: Num rows: 15 Data size: 162 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col0,_col3))
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ predicate: (((((UDFToDouble(key) > 10.0) and (UDFToDouble(key) > 20.0)) and (UDFToDouble(value) < 200.0)) and key is not null) and value is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 5 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string), _col1 (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col3
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: hash(_col0,_col3) (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 58 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
- Reducer 3
+ Reducer 2
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join17.q.out b/ql/src/test/results/clientpositive/spark/auto_join17.q.out
index 3144db6..bc492c9 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join17.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join17.q.out
@@ -28,15 +28,19 @@ STAGE PLANS:
Map 2
Map Operator Tree:
TableScan
- alias: src2
+ alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
@@ -52,28 +56,32 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col5, _col6
- input vertices:
- 1 Map 2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col5) (type: int), _col6 (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Local Work:
Map Reduce Local Work
@@ -101,9 +109,9 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.key1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.key2 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.value1 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.value2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: SELECT sum(hash(dest1.key1,dest1.value1,dest1.key2,dest1.value2)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join18.q.out b/ql/src/test/results/clientpositive/spark/auto_join18.q.out
index f5a2227..eaef06c 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join18.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join18.q.out
@@ -45,11 +45,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(value)
- keys: key (type: string)
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -66,11 +66,11 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT value)
- keys: key (type: string), value (type: string)
+ aggregations: count(DISTINCT _col1)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -103,15 +103,19 @@ STAGE PLANS:
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col0,_col1,_col2,_col3))
- mode: hash
+ Select Operator
+ expressions: hash(_col0,_col1,_col2,_col3) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 4
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join18_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join18_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/auto_join18_multi_distinct.q.out
index 07c7aa5..df737c2 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join18_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join18_multi_distinct.q.out
@@ -47,11 +47,11 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(value)
- keys: key (type: string)
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -68,11 +68,11 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT value), count(DISTINCT key)
- keys: key (type: string), value (type: string)
+ aggregations: count(DISTINCT _col1), count(DISTINCT _col0)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -105,15 +105,19 @@ STAGE PLANS:
1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(hash(_col0,_col1,_col2,_col3,_col4))
- mode: hash
+ Select Operator
+ expressions: hash(_col0,_col1,_col2,_col3,_col4) (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 4
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join19.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join19.q.out b/ql/src/test/results/clientpositive/spark/auto_join19.q.out
index f2b0140..9e4fb8f 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join19.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join19.q.out
@@ -35,10 +35,14 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
@@ -54,28 +58,32 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col8
- input vertices:
- 1 Map 2
- Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col8 (type: string)
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col4
+ input vertices:
+ 1 Map 2
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col4 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/330c31ae/ql/src/test/results/clientpositive/spark/auto_join26.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/auto_join26.q.out b/ql/src/test/results/clientpositive/spark/auto_join26.q.out
index 58821e9..3c437a1 100644
--- a/ql/src/test/results/clientpositive/spark/auto_join26.q.out
+++ b/ql/src/test/results/clientpositive/spark/auto_join26.q.out
@@ -29,7 +29,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
alias: x
@@ -37,20 +37,24 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 2)
+ Reducer 2 <- Map 1 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 1
Map Operator Tree:
TableScan
alias: y
@@ -58,31 +62,39 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string)
outputColumnNames: _col0
- input vertices:
- 0 Map 1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 3
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Local Work:
Map Reduce Local Work
- Reducer 3
+ Reducer 2
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -128,7 +140,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Input: default@src1
POSTHOOK: Output: default@dest_j1
-POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, (src)y.null, ]
+POSTHOOK: Lineage: dest_j1.cnt EXPRESSION [(src1)x.null, ]
POSTHOOK: Lineage: dest_j1.key EXPRESSION [(src1)x.FieldSchema(name:key, type:string, comment:default), ]
PREHOOK: query: select * from dest_j1
PREHOOK: type: QUERY
[18/61] [abbrv] hive git commit: HIVE-10949 : Disable hive-minikdc
tests in Windows (Hari Subramaniyan, reviewed by Vaibhav Gumashta)
Posted by xu...@apache.org.
HIVE-10949 : Disable hive-minikdc tests in Windows (Hari Subramaniyan, reviewed by Vaibhav Gumashta)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d0a9e789
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d0a9e789
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d0a9e789
Branch: refs/heads/spark
Commit: d0a9e789baca5140f660594e83b5b86b7df91c12
Parents: 5fff823
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Wed Jun 10 16:08:11 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Wed Jun 10 16:08:11 2015 -0700
----------------------------------------------------------------------
itests/hive-minikdc/pom.xml | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d0a9e789/itests/hive-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-minikdc/pom.xml b/itests/hive-minikdc/pom.xml
index 0fc7039..3098dac 100644
--- a/itests/hive-minikdc/pom.xml
+++ b/itests/hive-minikdc/pom.xml
@@ -29,6 +29,7 @@
<properties>
<hive.path.to.root>../..</hive.path.to.root>
+ <exclude.tests>None</exclude.tests>
</properties>
<dependencies>
@@ -265,6 +266,16 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <activation>
+ <os>
+ <family>Windows</family>
+ </os>
+ </activation>
+ <properties>
+ <exclude.tests>**/*.java</exclude.tests>
+ </properties>
+ </profile>
</profiles>
<build>
@@ -275,6 +286,15 @@
<inherited>true</inherited>
<extensions>true</extensions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>${exclude.tests}</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
[45/61] [abbrv] hive git commit: HIVE-11035: PPD: Orc Split
elimination fails because filterColumns=[-1] (Prasanth Jayachandran reviewed
by Gopal V)
Posted by xu...@apache.org.
HIVE-11035: PPD: Orc Split elimination fails because filterColumns=[-1] (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a511eb9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a511eb9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a511eb9
Branch: refs/heads/spark
Commit: 9a511eb97270a3a0d8bf3504ff3e884d5871628f
Parents: 03c62ce
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Jun 17 23:44:55 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Jun 17 23:44:55 2015 -0700
----------------------------------------------------------------------
.../hive/serde2/ColumnProjectionUtils.java | 29 +++-----------------
1 file changed, 4 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9a511eb9/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
index 08a4e99..10086c5 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
/**
@@ -38,6 +39,7 @@ public final class ColumnProjectionUtils {
public static final String READ_COLUMN_NAMES_CONF_STR = "hive.io.file.readcolumn.names";
private static final String READ_COLUMN_IDS_CONF_STR_DEFAULT = "";
private static final boolean READ_ALL_COLUMNS_DEFAULT = true;
+ private static final Joiner CSV_JOINER = Joiner.on(",").skipNulls();
/**
* @deprecated for backwards compatibility with <= 0.12, use setReadAllColumns
@@ -109,31 +111,8 @@ public final class ColumnProjectionUtils {
public static void appendReadColumns(
StringBuilder readColumnsBuffer, StringBuilder readColumnNamesBuffer, List<Integer> ids,
List<String> names) {
- appendReadColumns(readColumnsBuffer, ids);
- appendReadColumnNames(readColumnNamesBuffer, names);
- }
-
- public static void appendReadColumns(StringBuilder readColumnsBuffer, List<Integer> ids) {
- String id = toReadColumnIDString(ids);
- String newConfStr = id;
- if (readColumnsBuffer.length() > 0) {
- readColumnsBuffer.append(StringUtils.COMMA_STR).append(newConfStr);
- }
- if (readColumnsBuffer.length() == 0) {
- readColumnsBuffer.append(READ_COLUMN_IDS_CONF_STR_DEFAULT);
- }
- }
-
- private static void appendReadColumnNames(StringBuilder readColumnNamesBuffer, List<String> cols) {
- boolean first = readColumnNamesBuffer.length() > 0;
- for(String col: cols) {
- if (first) {
- first = false;
- } else {
- readColumnNamesBuffer.append(',');
- }
- readColumnNamesBuffer.append(col);
- }
+ CSV_JOINER.appendTo(readColumnsBuffer, ids);
+ CSV_JOINER.appendTo(readColumnNamesBuffer, names);
}
/**
[35/61] [abbrv] hive git commit: HIVE-10940 :
HiveInputFormat::pushFilters serializes PPD objects for each getRecordReader
call (Sergey Shelukhin, reviewed by Gopal V)
Posted by xu...@apache.org.
HIVE-10940 : HiveInputFormat::pushFilters serializes PPD objects for each getRecordReader call (Sergey Shelukhin, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d6db7058
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d6db7058
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d6db7058
Branch: refs/heads/spark
Commit: d6db70586291d7bbf213975a16115f245e17bd68
Parents: d8059d8
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jun 15 17:08:18 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jun 15 17:08:18 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/HiveInputFormat.java | 38 +++++++++++---------
.../hadoop/hive/ql/plan/TableScanDesc.java | 20 ++++++++++-
2 files changed, 41 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d6db7058/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 54e1d4e..2ff3951 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -450,25 +450,31 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return;
}
- Serializable filterObject = scanDesc.getFilterObject();
- if (filterObject != null) {
- jobConf.set(
- TableScanDesc.FILTER_OBJECT_CONF_STR,
- Utilities.serializeObject(filterObject));
+ String serializedFilterObj = scanDesc.getSerializedFilterObject();
+ String serializedFilterExpr = scanDesc.getSerializedFilterExpr();
+ boolean hasObj = serializedFilterObj != null, hasExpr = serializedFilterExpr != null;
+ if (!hasObj) {
+ Serializable filterObject = scanDesc.getFilterObject();
+ if (filterObject != null) {
+ serializedFilterObj = Utilities.serializeObject(filterObject);
+ }
+ }
+ if (serializedFilterObj != null) {
+ jobConf.set(TableScanDesc.FILTER_OBJECT_CONF_STR, serializedFilterObj);
+ }
+ if (!hasExpr) {
+ serializedFilterExpr = Utilities.serializeExpression(filterExpr);
}
-
String filterText = filterExpr.getExprString();
- String filterExprSerialized = Utilities.serializeExpression(filterExpr);
if (LOG.isDebugEnabled()) {
- LOG.debug("Filter text = " + filterText);
- LOG.debug("Filter expression = " + filterExprSerialized);
- }
- jobConf.set(
- TableScanDesc.FILTER_TEXT_CONF_STR,
- filterText);
- jobConf.set(
- TableScanDesc.FILTER_EXPR_CONF_STR,
- filterExprSerialized);
+ LOG.debug("Pushdown initiated with filterText = " + filterText + ", filterExpr = "
+ + filterExpr + ", serializedFilterExpr = " + serializedFilterExpr + " ("
+ + (hasExpr ? "desc" : "new") + ")" + (serializedFilterObj == null ? "" :
+ (", serializedFilterObj = " + serializedFilterObj + " (" + (hasObj ? "desc" : "new")
+ + ")")));
+ }
+ jobConf.set(TableScanDesc.FILTER_TEXT_CONF_STR, filterText);
+ jobConf.set(TableScanDesc.FILTER_EXPR_CONF_STR, serializedFilterExpr);
}
protected void pushProjectionsAndFilters(JobConf jobConf, Class inputFormatClass,
http://git-wip-us.apache.org/repos/asf/hive/blob/d6db7058/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 6282380..9e9a2a2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.parse.TableSample;
@@ -72,6 +73,9 @@ public class TableScanDesc extends AbstractOperatorDesc {
private ExprNodeGenericFuncDesc filterExpr;
private transient Serializable filterObject;
+ private String serializedFilterExpr;
+ private String serializedFilterObject;
+
// Both neededColumnIDs and neededColumns should never be null.
// When neededColumnIDs is an empty list,
@@ -101,7 +105,6 @@ public class TableScanDesc extends AbstractOperatorDesc {
private transient final Table tableMetadata;
-
public TableScanDesc() {
this(null, null);
}
@@ -144,7 +147,11 @@ public class TableScanDesc extends AbstractOperatorDesc {
}
public void setFilterExpr(ExprNodeGenericFuncDesc filterExpr) {
+ // TODO: we could avoid serialization if it's the same expr. Check?
this.filterExpr = filterExpr;
+ if (filterExpr != null) {
+ serializedFilterExpr = Utilities.serializeExpression(filterExpr);
+ }
}
public Serializable getFilterObject() {
@@ -153,6 +160,9 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setFilterObject(Serializable filterObject) {
this.filterObject = filterObject;
+ if (filterObject != null) {
+ serializedFilterObject = Utilities.serializeObject(filterObject);
+ }
}
public void setNeededColumnIDs(List<Integer> neededColumnIDs) {
@@ -281,4 +291,12 @@ public class TableScanDesc extends AbstractOperatorDesc {
public void setTableSample(TableSample tableSample) {
this.tableSample = tableSample;
}
+
+ public String getSerializedFilterExpr() {
+ return serializedFilterExpr;
+ }
+
+ public String getSerializedFilterObject() {
+ return serializedFilterObject;
+ }
}
[34/61] [abbrv] hive git commit: HIVE-10907: Hive on Tez: Classcast
exception in some cases with SMB joins (Vikram Dixit K,
reviewed by Gunther Hagleitner and Sergey Shelukhin)
Posted by xu...@apache.org.
HIVE-10907: Hive on Tez: Classcast exception in some cases with SMB joins (Vikram Dixit K, reviewed by Gunther Hagleitner and Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/71c36445
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/71c36445
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/71c36445
Branch: refs/heads/spark
Commit: 71c36445b85688fda6c4e7c4cca72d75bd07425f
Parents: 4427935
Author: vikram <vi...@hortonworks.com>
Authored: Mon Jun 15 16:59:15 2015 -0700
Committer: vikram <vi...@hortonworks.com>
Committed: Mon Jun 15 16:59:15 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 21 ++
ql/src/test/queries/clientpositive/tez_smb_1.q | 31 +++
.../results/clientpositive/tez/tez_smb_1.q.out | 278 +++++++++++++++++++
3 files changed, 330 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/71c36445/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index bcffdbc..5a87bd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.MuxOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TezDummyStoreOperator;
import org.apache.hadoop.hive.ql.lib.Node;
@@ -369,6 +370,26 @@ public class ConvertJoinMapJoin implements NodeProcessor {
(ReduceSinkOperator) joinOp.getParentOperators().get(bigTablePosition);
int numBuckets = bigTableRS.getParentOperators().get(0).getOpTraits().getNumBuckets();
+ int size = -1;
+ for (Operator<?> parentOp : joinOp.getParentOperators()) {
+ // each side better have 0 or more RS. if either side is unbalanced, cannot convert.
+ // This is a workaround for now. Right fix would be to refactor code in the
+ // MapRecordProcessor and ReduceRecordProcessor with respect to the sources.
+ Set<ReduceSinkOperator> set =
+ OperatorUtils.findOperatorsUpstream(parentOp.getParentOperators(),
+ ReduceSinkOperator.class);
+ if (size < 0) {
+ size = set.size();
+ continue;
+ }
+
+ if (((size > 0) && (set.size() > 0)) || ((size == 0) && (set.size() == 0))) {
+ continue;
+ } else {
+ return false;
+ }
+ }
+
// the sort and bucket cols have to match on both sides for this
// transformation of the join operation
for (Operator<? extends OperatorDesc> parentOp : joinOp.getParentOperators()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/71c36445/ql/src/test/queries/clientpositive/tez_smb_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_smb_1.q b/ql/src/test/queries/clientpositive/tez_smb_1.q
index 580672f..62a415b 100644
--- a/ql/src/test/queries/clientpositive/tez_smb_1.q
+++ b/ql/src/test/queries/clientpositive/tez_smb_1.q
@@ -34,6 +34,37 @@ set hive.auto.convert.join.noconditionaltask.size=500;
explain
select count(*) from tab s1 join tab s3 on s1.key=s3.key;
+set hive.convert.join.bucket.mapjoin.tez = false;
+explain
+select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id;
+
+select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id;
+
+explain
+select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id;
+
+select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id;
+
set hive.auto.convert.join=false;
explain
http://git-wip-us.apache.org/repos/asf/hive/blob/71c36445/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out b/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
index d970bd9..e60d5af 100644
--- a/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/tez_smb_1.q.out
@@ -179,6 +179,284 @@ STAGE PLANS:
PREHOOK: query: explain
select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: vt1
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col6
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col0 = _col6) (type: boolean)
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from
+tab vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.key=vt2.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+480
+PREHOOK: query: explain
+select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: vt1
+ Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 key (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col1 = _col0) (type: boolean)
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 137 Data size: 1455 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+join
+tab vt1
+where vt1.key=vt2.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+480
+PREHOOK: query: explain
+select count(*) from
(select rt1.id from
(select t1.key as id, t1.value as od from tab t1 order by id, od) rt1) vt1
join