You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/25 23:28:31 UTC
[01/50] [abbrv] hive git commit: HIVE-11451: SemanticAnalyzer throws
IndexOutOfBounds Exception (Aihua Xu, reviewed by Chao Sun)
Repository: hive
Updated Branches:
refs/heads/llap b97a07688 -> 88248fb6b
HIVE-11451: SemanticAnalyzer throws IndexOutOfBounds Exception (Aihua Xu, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db46e6e8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db46e6e8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db46e6e8
Branch: refs/heads/llap
Commit: db46e6e8ccc4ee86fbd074dbc3d2e4fa7d88ce25
Parents: bd90fc3
Author: Aihua Xu <ai...@gmail.com>
Authored: Thu Aug 13 10:55:30 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Thu Aug 13 11:24:31 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 15 +++++++++------
.../clientnegative/mismatch_columns_insertion.q | 4 ++++
.../clientnegative/mismatch_columns_insertion.q.out | 9 +++++++++
3 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 5ea6f3f..b809a23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6724,6 +6724,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
outColumnCnt += dpCtx.getNumDPCols();
}
+ // The numbers of input columns and output columns should match for regular query
+ if (!updating() && !deleting() && inColumnCnt != outColumnCnt) {
+ String reason = "Table " + dest + " has " + outColumnCnt
+ + " columns, but query has " + inColumnCnt + " columns.";
+ throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg(
+ qb.getParseInfo().getDestForClause(dest), reason));
+ }
+
// Check column types
boolean converted = false;
int columnNumber = tableFields.size();
@@ -6830,12 +6838,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
dpCtx.mapInputToDP(rowFields.subList(tableFields.size() + 1, rowFields.size()));
}
} else {
- if (inColumnCnt != outColumnCnt) {
- String reason = "Table " + dest + " has " + outColumnCnt
- + " columns, but query has " + inColumnCnt + " columns.";
- throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg(
- qb.getParseInfo().getDestForClause(dest), reason));
- } else if (dynPart && dpCtx != null) {
+ if (dynPart && dpCtx != null) {
// create the mapping from input ExprNode to dest table DP column
dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size()));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q b/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
new file mode 100644
index 0000000..dad1ec1
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
@@ -0,0 +1,4 @@
+
+create table mismatch_columns(key string, value string);
+
+insert overwrite table mismatch_columns select key from srcpart where ds is not null;
http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out b/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
new file mode 100644
index 0000000..831af12
--- /dev/null
+++ b/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
@@ -0,0 +1,9 @@
+PREHOOK: query: create table mismatch_columns(key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mismatch_columns
+POSTHOOK: query: create table mismatch_columns(key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mismatch_columns
+FAILED: SemanticException [Error 10044]: Line 3:23 Cannot insert into target table because column number/types are different 'mismatch_columns': Table insclause-0 has 2 columns, but query has 1 columns.
[11/50] [abbrv] hive git commit: HIVE-11562: Typo in hive-log4j2.xml
throws unknown level exception (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-11562: Typo in hive-log4j2.xml throws unknown level exception (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d307abbf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d307abbf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d307abbf
Branch: refs/heads/llap
Commit: d307abbf1093ffdc2599489bd6f67bdb8dcb3e14
Parents: c93d6c7
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 14:01:57 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 14:01:57 2015 -0700
----------------------------------------------------------------------
data/conf/hive-log4j2.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d307abbf/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index c9adfa2..a40be2c 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -70,7 +70,7 @@
<Loggers>
<Root level="${sys:hive.log.threshold}">
- <AppenderRef ref="${sys:hive.root.logger}" level="{sys:hive.log.level}"/>
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
<AppenderRef ref="EventCounter" />
</Root>
[06/50] [abbrv] hive git commit: HIVE-11493: Predicate with integer
column equals double evaluates to false (Pengcheng Xiong,
reviewed by Hari Sankar Sivarama Subramaniyan)
Posted by se...@apache.org.
HIVE-11493: Predicate with integer column equals double evaluates to false (Pengcheng Xiong, reviewed by Hari Sankar Sivarama Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8f1ae11
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8f1ae11
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8f1ae11
Branch: refs/heads/llap
Commit: b8f1ae110616a8fe162f79140c785f76be76fc67
Parents: 6e76291
Author: Pengcheng Xiong <px...@apache.org>
Authored: Thu Aug 13 21:01:54 2015 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Thu Aug 13 21:02:40 2015 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TypeCheckProcFactory.java | 2 +-
.../clientpositive/cast_tinyint_to_double.q | 7 ++++
.../clientpositive/cast_tinyint_to_double.q.out | 38 ++++++++++++++++++++
.../clientpositive/infer_const_type.q.out | 7 ++--
.../clientpositive/spark/vectorization_0.q.out | 2 +-
.../spark/vectorization_short_regress.q.out | 20 +++++------
.../clientpositive/tez/vectorization_0.q.out | 2 +-
.../tez/vectorization_short_regress.q.out | 20 +++++------
.../clientpositive/vectorization_0.q.out | 2 +-
.../vectorization_short_regress.q.out | 20 +++++------
10 files changed, 84 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index cd68f4e..ab5d006 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -1034,7 +1034,7 @@ public class TypeCheckProcFactory {
// we'll try again to convert it to double
// however, if we already tried this, or the column is NUMBER type and
// the operator is EQUAL, return false due to the type mismatch
- if (triedDouble ||
+ if (triedDouble &&
(genericUDF instanceof GenericUDFOPEqual
&& !columnType.equals(serdeConstants.STRING_TYPE_NAME))) {
return new ExprNodeConstantDesc(false);
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q b/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
new file mode 100644
index 0000000..59c5e89
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
@@ -0,0 +1,7 @@
+drop table t;
+CREATE TABLE t(c tinyint);
+insert overwrite table t select 10 from src limit 1;
+
+select * from t where c = 10.0;
+
+select * from t where c = -10.0;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out b/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
new file mode 100644
index 0000000..c29df65
--- /dev/null
+++ b/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
@@ -0,0 +1,38 @@
+PREHOOK: query: drop table t
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table t
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE t(c tinyint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: CREATE TABLE t(c tinyint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert overwrite table t select 10 from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@t
+POSTHOOK: query: insert overwrite table t select 10 from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.c EXPRESSION []
+PREHOOK: query: select * from t where c = 10.0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t where c = 10.0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+10
+PREHOOK: query: select * from t where c = -10.0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t where c = -10.0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/infer_const_type.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/infer_const_type.q.out b/ql/src/test/results/clientpositive/infer_const_type.q.out
index 6368e4a..05c6a45 100644
--- a/ql/src/test/results/clientpositive/infer_const_type.q.out
+++ b/ql/src/test/results/clientpositive/infer_const_type.q.out
@@ -102,6 +102,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@infertypes
#### A masked pattern was here ####
127 32767 12345 -12345 906.0 -307.0 1234
+WARNING: Comparing a bigint and a double may result in a loss of precision.
PREHOOK: query: -- all should return false as all numbers exceeed the largest number
-- which could be represented by the corresponding type
-- and string_col = long_const should return false
@@ -136,7 +137,7 @@ STAGE PLANS:
alias: infertypes
Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: false (type: boolean)
+ predicate: ((UDFToDouble(ti) = 128.0) or (UDFToDouble(si) = 32768.0) or (UDFToDouble(i) = 2.147483648E9) or (UDFToDouble(bi) = 9.223372036854776E18)) (type: boolean)
Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string)
@@ -156,6 +157,7 @@ STAGE PLANS:
Processor Tree:
ListSink
+WARNING: Comparing a bigint and a double may result in a loss of precision.
PREHOOK: query: SELECT * FROM infertypes WHERE
ti = '128' OR
si = 32768 OR
@@ -200,7 +202,7 @@ STAGE PLANS:
alias: infertypes
Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: false (type: boolean)
+ predicate: ((UDFToDouble(ti) = 127.0) or (UDFToDouble(si) = 327.0) or (UDFToDouble(i) = -100.0)) (type: boolean)
Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string)
@@ -234,6 +236,7 @@ POSTHOOK: query: SELECT * FROM infertypes WHERE
POSTHOOK: type: QUERY
POSTHOOK: Input: default@infertypes
#### A masked pattern was here ####
+127 32767 12345 -12345 906.0 -307.0 1234
PREHOOK: query: EXPLAIN SELECT * FROM infertypes WHERE
ti < '127.0' AND
i > '100.0' AND
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
index 90bcc1b..3ad059c 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
@@ -994,7 +994,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
+ predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index 6bd1bb2..3d17aba 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -2085,23 +2085,23 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
- Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
+ Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), cfloat (type: float)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
Execution mode: vectorized
Reducer 2
@@ -2111,25 +2111,25 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
index cfbe9ce..18e042d 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
@@ -994,7 +994,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
+ predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
index 5e0d42c..59b457a 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
@@ -2085,23 +2085,23 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), cfloat (type: float)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
Execution mode: vectorized
Reducer 2
@@ -2111,25 +2111,25 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_0.q.out b/ql/src/test/results/clientpositive/vectorization_0.q.out
index 3fab2ff..89163cd 100644
--- a/ql/src/test/results/clientpositive/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_0.q.out
@@ -1010,7 +1010,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
+ predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)
http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
index f2cb3ec..728f628 100644
--- a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
@@ -2031,23 +2031,23 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), cfloat (type: float)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
Execution mode: vectorized
Reduce Operator Tree:
@@ -2056,11 +2056,11 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -2075,16 +2075,16 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
[40/50] [abbrv] hive git commit: HIVE-11586:
ObjectInspectorFactory.getReflectionObjectInspector is not thread-safe (Jimmy,
reviewed by Szehon, Xuefu)
Posted by se...@apache.org.
HIVE-11586: ObjectInspectorFactory.getReflectionObjectInspector is not thread-safe (Jimmy, reviewed by Szehon, Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2d148ba
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2d148ba
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2d148ba
Branch: refs/heads/llap
Commit: e2d148ba83a81fb03303bd3120b693759a2cd0ff
Parents: 3c1eae0
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Mon Aug 17 10:08:09 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Sat Aug 22 11:16:59 2015 -0700
----------------------------------------------------------------------
.../objectinspector/ObjectInspectorFactory.java | 61 +++++++++++++----
.../ReflectionStructObjectInspector.java | 60 ++++++++++++++---
.../ThriftUnionObjectInspector.java | 28 +++++---
.../TestReflectionObjectInspectors.java | 71 +++++++++++++++++++-
4 files changed, 185 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e2d148ba/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
index 97bb715..2b3fded 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
@@ -23,6 +23,7 @@ import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -61,14 +62,38 @@ public final class ObjectInspectorFactory {
JAVA, THRIFT, PROTOCOL_BUFFERS, AVRO
};
- private static ConcurrentHashMap<Type, ObjectInspector> objectInspectorCache = new ConcurrentHashMap<Type, ObjectInspector>();
+ static ConcurrentHashMap<Type, ObjectInspector> objectInspectorCache = new ConcurrentHashMap<Type, ObjectInspector>();
public static ObjectInspector getReflectionObjectInspector(Type t,
ObjectInspectorOptions options) {
+ return getReflectionObjectInspector(t, options, true);
+ }
+
+ static ObjectInspector getReflectionObjectInspector(Type t,
+ ObjectInspectorOptions options, boolean ensureInited) {
ObjectInspector oi = objectInspectorCache.get(t);
if (oi == null) {
- oi = getReflectionObjectInspectorNoCache(t, options);
- objectInspectorCache.put(t, oi);
+ oi = getReflectionObjectInspectorNoCache(t, options, ensureInited);
+ ObjectInspector prev = objectInspectorCache.putIfAbsent(t, oi);
+ if (prev != null) {
+ oi = prev;
+ }
+ }
+ if (ensureInited && oi instanceof ReflectionStructObjectInspector) {
+ ReflectionStructObjectInspector soi = (ReflectionStructObjectInspector) oi;
+ synchronized (soi) {
+ HashSet<Type> checkedTypes = new HashSet<Type>();
+ while (!soi.isFullyInited(checkedTypes)) {
+ try {
+ // Wait for up to 3 seconds before checking if any init error.
+ // Init should be fast if no error, no need to make this configurable.
+ soi.wait(3000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted while waiting for "
+ + soi.getClass().getName() + " to initialize", e);
+ }
+ }
+ }
}
verifyObjectInspector(options, oi, ObjectInspectorOptions.JAVA, new Class[]{ThriftStructObjectInspector.class,
ProtocolBuffersStructObjectInspector.class});
@@ -88,10 +113,10 @@ public final class ObjectInspectorFactory {
* @param classes ObjectInspector should not be of these types
*/
private static void verifyObjectInspector(ObjectInspectorOptions option, ObjectInspector oi,
- ObjectInspectorOptions checkOption, Class[] classes) {
+ ObjectInspectorOptions checkOption, Class<?>[] classes) {
if (option.equals(checkOption)) {
- for (Class checkClass : classes) {
+ for (Class<?> checkClass : classes) {
if (oi.getClass().equals(checkClass)) {
throw new RuntimeException(
"Cannot call getObjectInspectorByReflection with more then one of " +
@@ -102,11 +127,11 @@ public final class ObjectInspectorFactory {
}
private static ObjectInspector getReflectionObjectInspectorNoCache(Type t,
- ObjectInspectorOptions options) {
+ ObjectInspectorOptions options, boolean ensureInited) {
if (t instanceof GenericArrayType) {
GenericArrayType at = (GenericArrayType) t;
return getStandardListObjectInspector(getReflectionObjectInspector(at
- .getGenericComponentType(), options));
+ .getGenericComponentType(), options, ensureInited));
}
if (t instanceof ParameterizedType) {
@@ -115,14 +140,14 @@ public final class ObjectInspectorFactory {
if (List.class.isAssignableFrom((Class<?>) pt.getRawType()) ||
Set.class.isAssignableFrom((Class<?>) pt.getRawType())) {
return getStandardListObjectInspector(getReflectionObjectInspector(pt
- .getActualTypeArguments()[0], options));
+ .getActualTypeArguments()[0], options, ensureInited));
}
// Map?
if (Map.class.isAssignableFrom((Class<?>) pt.getRawType())) {
return getStandardMapObjectInspector(getReflectionObjectInspector(pt
- .getActualTypeArguments()[0], options),
+ .getActualTypeArguments()[0], options, ensureInited),
getReflectionObjectInspector(pt.getActualTypeArguments()[1],
- options));
+ options, ensureInited));
}
// Otherwise convert t to RawType so we will fall into the following if
// block.
@@ -186,8 +211,20 @@ public final class ObjectInspectorFactory {
// put it into the cache BEFORE it is initialized to make sure we can catch
// recursive types.
- objectInspectorCache.put(t, oi);
- oi.init(c, options);
+ ReflectionStructObjectInspector prev =
+ (ReflectionStructObjectInspector) objectInspectorCache.putIfAbsent(t, oi);
+ if (prev != null) {
+ oi = prev;
+ } else {
+ try {
+ oi.init(t, c, options);
+ } finally {
+ if (!oi.inited) {
+ // Failed to init, remove it from cache
+ objectInspectorCache.remove(t, oi);
+ }
+ }
+ }
return oi;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2d148ba/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
index 78e6066..22f8051 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.hive.serde2.objectinspector;
import java.lang.reflect.Field;
+import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.util.ReflectionUtils;
@@ -81,6 +83,8 @@ public class ReflectionStructObjectInspector extends
Class<?> objectClass;
List<MyField> fields;
+ volatile boolean inited = false;
+ volatile Type type;
public Category getCategory() {
return Category.STRUCT;
@@ -113,12 +117,44 @@ public class ReflectionStructObjectInspector extends
}
/**
+ * Check if this inspector and all its field inspectors are initialized.
+ */
+ protected boolean isFullyInited(Set<Type> checkedTypes) {
+ if (type != null && // when type is not set, init hasn't been called yet
+ ObjectInspectorFactory.objectInspectorCache.get(type) != this) {
+ // This object should be the same as in cache, otherwise, it must be removed due to init error
+ throw new RuntimeException("Cached object inspector is gone while waiting for it to initialize");
+ }
+
+ if (!inited) {
+ return false;
+ }
+
+ // We don't want to check types already checked
+ checkedTypes.add(type);
+
+ // This inspector is initialized, we still need
+ // to check if all field inspectors are initialized
+ for (StructField field: getAllStructFieldRefs()) {
+ ObjectInspector oi = field.getFieldObjectInspector();
+ if (oi instanceof ReflectionStructObjectInspector) {
+ ReflectionStructObjectInspector soi = (ReflectionStructObjectInspector) oi;
+ if (!checkedTypes.contains(soi.type) && !soi.isFullyInited(checkedTypes)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
* This method is only intended to be used by Utilities class in this package.
* The reason that this method is not recursive by itself is because we want
* to allow recursive types.
*/
- protected void init(Class<?> objectClass,
+ protected void init(Type type, Class<?> objectClass,
ObjectInspectorFactory.ObjectInspectorOptions options) {
+ this.type = type;
verifyObjectClassType(objectClass);
this.objectClass = objectClass;
@@ -126,16 +162,20 @@ public class ReflectionStructObjectInspector extends
Field[] reflectionFields = ObjectInspectorUtils
.getDeclaredNonStaticFields(objectClass);
- fields = new ArrayList<MyField>(structFieldObjectInspectors.size());
- int used = 0;
- for (int i = 0; i < reflectionFields.length; i++) {
- if (!shouldIgnoreField(reflectionFields[i].getName())) {
- reflectionFields[i].setAccessible(true);
- fields.add(new MyField(i, reflectionFields[i], structFieldObjectInspectors
- .get(used++)));
+ synchronized (this) {
+ fields = new ArrayList<MyField>(structFieldObjectInspectors.size());
+ int used = 0;
+ for (int i = 0; i < reflectionFields.length; i++) {
+ if (!shouldIgnoreField(reflectionFields[i].getName())) {
+ reflectionFields[i].setAccessible(true);
+ fields.add(new MyField(i, reflectionFields[i], structFieldObjectInspectors
+ .get(used++)));
+ }
}
+ assert (fields.size() == structFieldObjectInspectors.size());
+ inited = true;
+ notifyAll();
}
- assert (fields.size() == structFieldObjectInspectors.size());
}
// ThriftStructObjectInspector will override and ignore __isset fields.
@@ -215,7 +255,7 @@ public class ReflectionStructObjectInspector extends
for (int i = 0; i < fields.length; i++) {
if (!shouldIgnoreField(fields[i].getName())) {
structFieldObjectInspectors.add(ObjectInspectorFactory.getReflectionObjectInspector(fields[i]
- .getGenericType(), options));
+ .getGenericType(), options, false));
}
}
return structFieldObjectInspectors;
http://git-wip-us.apache.org/repos/asf/hive/blob/e2d148ba/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
index 600abbb..8593a41 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
@@ -34,6 +34,7 @@ import com.google.common.primitives.UnsignedBytes;
* Always use the ObjectInspectorFactory to create new ObjectInspector objects,
* instead of directly creating an instance of this class.
*/
+@SuppressWarnings("unchecked")
public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector implements UnionObjectInspector {
private static final String FIELD_METADATA_MAP = "metaDataMap";
@@ -73,8 +74,10 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
* to allow recursive types.
*/
@Override
- protected void init(Class<?> objectClass,
+ protected void init(Type type, Class<?> objectClass,
ObjectInspectorFactory.ObjectInspectorOptions options) {
+ this.type = type;
+
verifyObjectClassType(objectClass);
this.objectClass = objectClass;
final Field fieldMetaData;
@@ -89,15 +92,18 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
try {
final Map<? extends TFieldIdEnum, FieldMetaData> fieldMap = (Map<? extends TFieldIdEnum, FieldMetaData>) fieldMetaData.get(null);
- fields = new ArrayList<StandardStructObjectInspector.MyField>(fieldMap.size());
- this.ois = new ArrayList<ObjectInspector>();
- for(Map.Entry<? extends TFieldIdEnum, FieldMetaData> metadata : fieldMap.entrySet()) {
- int fieldId = metadata.getKey().getThriftFieldId();
- String fieldName = metadata.getValue().fieldName;
- final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, fieldName);
- final ObjectInspector reflectionObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(fieldType, options);
- fields.add(new StandardStructObjectInspector.MyField(fieldId, fieldName, reflectionObjectInspector));
- this.ois.add(reflectionObjectInspector);
+ synchronized (this) {
+ fields = new ArrayList<StandardStructObjectInspector.MyField>(fieldMap.size());
+ this.ois = new ArrayList<ObjectInspector>();
+ for(Map.Entry<? extends TFieldIdEnum, FieldMetaData> metadata : fieldMap.entrySet()) {
+ int fieldId = metadata.getKey().getThriftFieldId();
+ String fieldName = metadata.getValue().fieldName;
+ final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, fieldName);
+ final ObjectInspector reflectionObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(fieldType, options, false);
+ fields.add(new StandardStructObjectInspector.MyField(fieldId, fieldName, reflectionObjectInspector));
+ this.ois.add(reflectionObjectInspector);
+ }
+ inited = true;
}
} catch (IllegalAccessException e) {
throw new RuntimeException("Unable to find field metadata for thrift union field ", e);
@@ -110,7 +116,7 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
}
@Override
- public List<? extends StructField> getAllStructFieldRefs() {
+ public synchronized List<? extends StructField> getAllStructFieldRefs() {
return fields;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2d148ba/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java
index e2408c6..c14366a 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java
@@ -17,15 +17,24 @@
*/
package org.apache.hadoop.hive.serde2.objectinspector;
+import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
-import junit.framework.TestCase;
-
+import org.apache.commons.lang.mutable.MutableObject;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.thrift.test.Complex;
+
+import junit.framework.TestCase;
/**
* TestReflectionObjectInspectors.
@@ -100,4 +109,62 @@ public class TestReflectionObjectInspectors extends TestCase {
throw e;
}
}
+
+ public void testObjectInspectorThreadSafety() throws InterruptedException {
+ final int workerCount = 5; // 5 workers to run getReflectionObjectInspector concurrently
+ final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(workerCount);
+ final MutableObject exception = new MutableObject();
+ Thread runner = new Thread(new Runnable() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void run() {
+ Future<ObjectInspector>[] results = (Future<ObjectInspector>[])new Future[workerCount];
+ ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[] types =
+ (ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[])new ObjectPair[] {
+ new ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>(Complex.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.THRIFT),
+ new ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>(MyStruct.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA),
+ };
+ try {
+ for (int i = 0; i < 20; i++) { // repeat 20 times
+ for (final ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions> t: types) {
+ ObjectInspectorFactory.objectInspectorCache.clear();
+ for (int k = 0; k < workerCount; k++) {
+ results[k] = executorService.schedule(new Callable<ObjectInspector>() {
+ @Override
+ public ObjectInspector call() throws Exception {
+ return ObjectInspectorFactory.getReflectionObjectInspector(
+ t.getFirst(), t.getSecond());
+ }
+ }, 50, TimeUnit.MILLISECONDS);
+ }
+ ObjectInspector oi = results[0].get();
+ for (int k = 1; k < workerCount; k++) {
+ assertEquals(oi, results[k].get());
+ }
+ }
+ }
+ } catch (Throwable e) {
+ exception.setValue(e);
+ }
+ }
+ });
+ try {
+ runner.start();
+ long endTime = System.currentTimeMillis() + 300000; // timeout in 5 minutes
+ while (runner.isAlive()) {
+ if (System.currentTimeMillis() > endTime) {
+ runner.interrupt(); // Interrupt the runner thread
+ fail("Timed out waiting for the runner to finish");
+ }
+ runner.join(10000);
+ }
+ if (exception.getValue() != null) {
+ fail("Got exception: " + exception.getValue());
+ }
+ } finally {
+ executorService.shutdownNow();
+ }
+ }
}
[30/50] [abbrv] hive git commit: HIVE-11592: ORC metadata section can
sometimes exceed protobuf message size limit (Prasanth Jayachandran reviewed
by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11592: ORC metadata section can sometimes exceed protobuf message size limit (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8b02610
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8b02610
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8b02610
Branch: refs/heads/llap
Commit: f8b02610d745e63e3d596d7532e84e49eedbd62e
Parents: 2688b68
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Aug 19 11:40:52 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Aug 19 11:40:52 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 34 +++++++++++++++++++-
1 file changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f8b02610/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index a6448b6..c990d85 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -46,12 +46,15 @@ import org.apache.hadoop.io.Text;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.InvalidProtocolBufferException;
public class ReaderImpl implements Reader {
private static final Log LOG = LogFactory.getLog(ReaderImpl.class);
private static final int DIRECTORY_SIZE_GUESS = 16 * 1024;
+ private static final int DEFAULT_PROTOBUF_MESSAGE_LIMIT = 64 << 20; // 64MB
+ private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB
protected final FileSystem fileSystem;
protected final Path path;
@@ -468,7 +471,36 @@ public class ReaderImpl implements Reader {
InputStream instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
- this.metadata = OrcProto.Metadata.parseFrom(instream);
+ CodedInputStream in = CodedInputStream.newInstance(instream);
+ int msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT;
+ OrcProto.Metadata meta = null;
+ do {
+ try {
+ in.setSizeLimit(msgLimit);
+ meta = OrcProto.Metadata.parseFrom(in);
+ } catch (InvalidProtocolBufferException e) {
+ if (e.getMessage().contains("Protocol message was too large")) {
+ LOG.warn("Metadata section is larger than " + msgLimit + " bytes. Increasing the max" +
+ " size of the coded input stream." );
+
+ msgLimit = msgLimit << 1;
+ if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) {
+ LOG.error("Metadata section exceeds max protobuf message size of " +
+ PROTOBUF_MESSAGE_MAX_LIMIT + " bytes.");
+ throw e;
+ }
+
+ // we must have failed in the middle of reading instream and instream doesn't support
+ // resetting the stream
+ instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
+ new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
+ in = CodedInputStream.newInstance(instream);
+ } else {
+ throw e;
+ }
+ }
+ } while (meta == null);
+ this.metadata = meta;
footerBuffer.position(position + metadataSize);
footerBuffer.limit(position + metadataSize + footerBufferSize);
[25/50] [abbrv] hive git commit: HIVE-11429: Increase default JDBC
result set fetch size (# rows it fetches in one RPC call) to 1000 from 50
(Vaibhav Gumashta reviewed by Thejas Nair)
Posted by se...@apache.org.
HIVE-11429: Increase default JDBC result set fetch size (# rows it fetches in one RPC call) to 1000 from 50 (Vaibhav Gumashta reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5edbf31a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5edbf31a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5edbf31a
Branch: refs/heads/llap
Commit: 5edbf31a755668b213c2a21fb8a4a2e902e081f6
Parents: 3b6825b
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Tue Aug 18 02:12:35 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Tue Aug 18 02:12:35 2015 -0700
----------------------------------------------------------------------
jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5edbf31a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index 170fc53..d4041bb 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -59,7 +59,7 @@ public class HiveStatement implements java.sql.Statement {
private TOperationHandle stmtHandle = null;
private final TSessionHandle sessHandle;
Map<String,String> sessConf = new HashMap<String,String>();
- private int fetchSize = 50;
+ private int fetchSize = 1000;
private boolean isScrollableResultset = false;
/**
* We need to keep a reference to the result set to support the following:
[15/50] [abbrv] hive git commit: HIVE-11570: Fix PTest2
log4j2.version (Gopal V, via Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11570: Fix PTest2 log4j2.version (Gopal V, via Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/17e95c7c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/17e95c7c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/17e95c7c
Branch: refs/heads/llap
Commit: 17e95c7c761525a4d7596099cfca179db2a19a20
Parents: e8b2c60
Author: Gopal V <go...@apache.org>
Authored: Fri Aug 14 15:22:44 2015 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri Aug 14 15:22:44 2015 -0700
----------------------------------------------------------------------
testutils/ptest2/pom.xml | 16 +---------------
1 file changed, 1 insertion(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/17e95c7c/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index 2cf7f45..fade125 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -26,6 +26,7 @@ limitations under the License.
<name>hive-ptest</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <log4j2.version>2.3</log4j2.version>
</properties>
<repositories>
@@ -84,21 +85,6 @@ limitations under the License.
<version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>1.2.17</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.2.5</version>
[48/50] [abbrv] hive git commit: HIVE-11581: HiveServer2 should store
connection params in ZK when using dynamic service discovery for simpler
client connection string (Vaibhav Gumashta reviewed by Thejas Nair)
Posted by se...@apache.org.
HIVE-11581: HiveServer2 should store connection params in ZK when using dynamic service discovery for simpler client connection string (Vaibhav Gumashta reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0e54991d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0e54991d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0e54991d
Branch: refs/heads/llap
Commit: 0e54991d897c9acc26b015b6df82b44c0c90c6fb
Parents: dd2bdfc
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Mon Aug 24 17:14:27 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Mon Aug 24 17:15:22 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 72 ++++++------
.../org/apache/hive/jdbc/HiveConnection.java | 4 +-
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 117 +++++++------------
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 104 ++++++++++++++---
.../apache/hive/service/server/HiveServer2.java | 74 +++++++++++-
5 files changed, 239 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index da171b1..8706a2d 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1683,22 +1683,6 @@ public class HiveConf extends Configuration {
"to construct a list exception handlers to handle exceptions thrown\n" +
"by record readers"),
- // operation log configuration
- HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true,
- "When true, HS2 will save operation logs and make them available for clients"),
- HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location",
- "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator +
- "operation_logs",
- "Top level directory where operation logs are stored if logging functionality is enabled"),
- HIVE_SERVER2_LOGGING_OPERATION_LEVEL("hive.server2.logging.operation.level", "EXECUTION",
- new StringSet("NONE", "EXECUTION", "PERFORMANCE", "VERBOSE"),
- "HS2 operation logging mode available to clients to be set at session level.\n" +
- "For this to work, hive.server2.logging.operation.enabled should be set to true.\n" +
- " NONE: Ignore any logging\n" +
- " EXECUTION: Log completion of tasks\n" +
- " PERFORMANCE: Execution + Performance logs \n" +
- " VERBOSE: All logs" ),
- HIVE_SERVER2_METRICS_ENABLED("hive.server2.metrics.enabled", false, "Enable metrics on the HiveServer2."),
// logging configuration
HIVE_LOG4J_FILE("hive.log4j.file", "",
"Hive log4j configuration file.\n" +
@@ -1790,6 +1774,7 @@ public class HiveConf extends Configuration {
"hive.zookeeper.quorum in their connection string."),
HIVE_SERVER2_ZOOKEEPER_NAMESPACE("hive.server2.zookeeper.namespace", "hiveserver2",
"The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery."),
+
// HiveServer2 global init file location
HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
"Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" +
@@ -1801,6 +1786,39 @@ public class HiveConf extends Configuration {
HIVE_SERVER2_PARALLEL_COMPILATION("hive.driver.parallel.compilation", false, "Whether to\n" +
"enable parallel compilation between sessions on HiveServer2. The default is false."),
+ // Tez session settings
+ HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
+ "A list of comma separated values corresponding to YARN queues of the same name.\n" +
+ "When HiveServer2 is launched in Tez mode, this configuration needs to be set\n" +
+ "for multiple Tez sessions to run in parallel on the cluster."),
+ HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE("hive.server2.tez.sessions.per.default.queue", 1,
+ "A positive integer that determines the number of Tez sessions that should be\n" +
+ "launched on each of the queues specified by \"hive.server2.tez.default.queues\".\n" +
+ "Determines the parallelism on each queue."),
+ HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions", false,
+ "This flag is used in HiveServer2 to enable a user to use HiveServer2 without\n" +
+ "turning on Tez for HiveServer2. The user could potentially want to run queries\n" +
+ "over Tez without the pool of sessions."),
+
+ // Operation log configuration
+ HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true,
+ "When true, HS2 will save operation logs and make them available for clients"),
+ HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location",
+ "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator +
+ "operation_logs",
+ "Top level directory where operation logs are stored if logging functionality is enabled"),
+ HIVE_SERVER2_LOGGING_OPERATION_LEVEL("hive.server2.logging.operation.level", "EXECUTION",
+ new StringSet("NONE", "EXECUTION", "PERFORMANCE", "VERBOSE"),
+ "HS2 operation logging mode available to clients to be set at session level.\n" +
+ "For this to work, hive.server2.logging.operation.enabled should be set to true.\n" +
+ " NONE: Ignore any logging\n" +
+ " EXECUTION: Log completion of tasks\n" +
+ " PERFORMANCE: Execution + Performance logs \n" +
+ " VERBOSE: All logs" ),
+
+ // Enable metric collection for HiveServer2
+ HIVE_SERVER2_METRICS_ENABLED("hive.server2.metrics.enabled", false, "Enable metrics on the HiveServer2."),
+
// http (over thrift) transport settings
HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001,
"Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'http'."),
@@ -1816,7 +1834,7 @@ public class HiveConf extends Configuration {
"Keepalive time for an idle http worker thread. When the number of workers exceeds min workers, " +
"excessive threads are killed after this time interval."),
- // Cookie based authentication
+ // Cookie based authentication when using HTTP Transport
HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED("hive.server2.thrift.http.cookie.auth.enabled", true,
"When true, HiveServer2 in HTTP transport mode, will use cookie based authentication mechanism."),
HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE("hive.server2.thrift.http.cookie.max.age", "86400s",
@@ -1963,6 +1981,8 @@ public class HiveConf extends Configuration {
" HIVE : Exposes Hive's native table types like MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW\n" +
" CLASSIC : More generic types like TABLE and VIEW"),
HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", "", ""),
+
+ // SSL settings
HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", false,
"Set this to true for using SSL encryption in HiveServer2."),
HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", "",
@@ -1983,9 +2003,6 @@ public class HiveConf extends Configuration {
"Comma separated list of udfs names. These udfs will not be allowed in queries." +
" The udf black list takes precedence over udf white list"),
- HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
- "Comma separated list of non-SQL Hive commands users are authorized to execute"),
-
HIVE_SERVER2_SESSION_CHECK_INTERVAL("hive.server2.session.check.interval", "6h",
new TimeValidator(TimeUnit.MILLISECONDS, 3000l, true, null, false),
"The check interval for session/operation timeout, which can be disabled by setting to zero or negative value."),
@@ -2002,6 +2019,8 @@ public class HiveConf extends Configuration {
" This setting takes effect only if session idle timeout (hive.server2.idle.session.timeout) and checking\n" +
"(hive.server2.session.check.interval) are enabled."),
+ HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
+ "Comma separated list of non-SQL Hive commands users are authorized to execute"),
HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list",
"hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
"Comma separated list of configuration options which are immutable at runtime"),
@@ -2127,19 +2146,6 @@ public class HiveConf extends Configuration {
HIVECOUNTERGROUP("hive.counters.group.name", "HIVE",
"The name of counter group for internal Hive variables (CREATED_FILE, FATAL_ERROR, etc.)"),
- HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
- "A list of comma separated values corresponding to YARN queues of the same name.\n" +
- "When HiveServer2 is launched in Tez mode, this configuration needs to be set\n" +
- "for multiple Tez sessions to run in parallel on the cluster."),
- HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE("hive.server2.tez.sessions.per.default.queue", 1,
- "A positive integer that determines the number of Tez sessions that should be\n" +
- "launched on each of the queues specified by \"hive.server2.tez.default.queues\".\n" +
- "Determines the parallelism on each queue."),
- HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions", false,
- "This flag is used in HiveServer2 to enable a user to use HiveServer2 without\n" +
- "turning on Tez for HiveServer2. The user could potentially want to run queries\n" +
- "over Tez without the pool of sessions."),
-
HIVE_QUOTEDID_SUPPORT("hive.support.quoted.identifiers", "column",
new StringSet("none", "column"),
"Whether to use quoted identifier. 'none' or 'column' can be used. \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index a9dac03..ba971fd 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -211,13 +211,13 @@ public class HiveConnection implements java.sql.Connection {
break;
} catch (TTransportException e) {
LOG.info("Could not open client transport with JDBC Uri: " + jdbcUriString);
- // We'll retry till we exhaust all HiveServer2 uris from ZooKeeper
+ // We'll retry till we exhaust all HiveServer2 nodes from ZooKeeper
if ((sessConfMap.get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE) != null)
&& (JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER.equalsIgnoreCase(sessConfMap
.get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE)))) {
try {
// Update jdbcUriString, host & port variables in connParams
- // Throw an exception if all HiveServer2 uris have been exhausted,
+ // Throw an exception if all HiveServer2 nodes have been exhausted,
// or if we're unable to connect to ZooKeeper.
Utils.updateConnParamsFromZooKeeper(connParams);
} catch (ZooKeeperHiveClientException ze) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/Utils.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index 0e4693b..d8368a4 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -19,7 +19,6 @@
package org.apache.hive.jdbc;
import java.net.URI;
-import java.net.URISyntaxException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -37,22 +36,22 @@ import org.apache.hive.service.cli.thrift.TStatusCode;
import org.apache.http.client.CookieStore;
import org.apache.http.cookie.Cookie;
-public class Utils {
- public static final Log LOG = LogFactory.getLog(Utils.class.getName());
+class Utils {
+ static final Log LOG = LogFactory.getLog(Utils.class.getName());
/**
* The required prefix for the connection URL.
*/
- public static final String URL_PREFIX = "jdbc:hive2://";
+ static final String URL_PREFIX = "jdbc:hive2://";
/**
* If host is provided, without a port.
*/
- public static final String DEFAULT_PORT = "10000";
+ static final String DEFAULT_PORT = "10000";
/**
* Hive's default database name
*/
- public static final String DEFAULT_DATABASE = "default";
+ static final String DEFAULT_DATABASE = "default";
private static final String URI_JDBC_PREFIX = "jdbc:";
@@ -63,7 +62,7 @@ public class Utils {
static final String HIVE_SERVER2_RETRY_TRUE = "true";
static final String HIVE_SERVER2_RETRY_FALSE = "false";
- public static class JdbcConnectionParams {
+ static class JdbcConnectionParams {
// Note on client side parameter naming convention:
// Prefer using a shorter camelCase param name instead of using the same name as the
// corresponding
@@ -129,7 +128,7 @@ public class Utils {
static final String SSL_TRUST_STORE_TYPE = "JKS";
private String host = null;
- private int port;
+ private int port = 0;
private String jdbcUriString;
private String dbName = DEFAULT_DATABASE;
private Map<String,String> hiveConfs = new LinkedHashMap<String,String>();
@@ -238,17 +237,17 @@ public class Utils {
}
// Verify success or success_with_info status, else throw SQLException
- public static void verifySuccessWithInfo(TStatus status) throws SQLException {
+ static void verifySuccessWithInfo(TStatus status) throws SQLException {
verifySuccess(status, true);
}
// Verify success status, else throw SQLException
- public static void verifySuccess(TStatus status) throws SQLException {
+ static void verifySuccess(TStatus status) throws SQLException {
verifySuccess(status, false);
}
// Verify success and optionally with_info status, else throw SQLException
- public static void verifySuccess(TStatus status, boolean withInfo) throws SQLException {
+ static void verifySuccess(TStatus status, boolean withInfo) throws SQLException {
if (status.getStatusCode() == TStatusCode.SUCCESS_STATUS ||
(withInfo && status.getStatusCode() == TStatusCode.SUCCESS_WITH_INFO_STATUS)) {
return;
@@ -279,7 +278,7 @@ public class Utils {
* @return
* @throws SQLException
*/
- public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException,
+ static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException,
SQLException, ZooKeeperHiveClientException {
JdbcConnectionParams connParams = new JdbcConnectionParams();
@@ -383,7 +382,6 @@ public class Utils {
newUsage = usageUrlBase + JdbcConnectionParams.HTTP_PATH + "=<http_path_value>";
handleParamDeprecation(connParams.getHiveConfs(), connParams.getSessionVars(),
JdbcConnectionParams.HTTP_PATH_DEPRECATED, JdbcConnectionParams.HTTP_PATH, newUsage);
-
// Extract host, port
if (connParams.isEmbeddedMode()) {
// In case of embedded mode we were supplied with an empty authority.
@@ -391,23 +389,15 @@ public class Utils {
connParams.setHost(jdbcURI.getHost());
connParams.setPort(jdbcURI.getPort());
} else {
- // Else substitute the dummy authority with a resolved one.
- // In case of dynamic service discovery using ZooKeeper, it picks a server uri from ZooKeeper
- String resolvedAuthorityString = resolveAuthority(connParams);
- LOG.info("Resolved authority: " + resolvedAuthorityString);
- uri = uri.replace(dummyAuthorityString, resolvedAuthorityString);
+ // Configure host, port and params from ZooKeeper if used,
+ // and substitute the dummy authority with a resolved one
+ configureConnParams(connParams);
+ // We check for invalid host, port while configuring connParams with configureConnParams()
+ String authorityStr = connParams.getHost() + ":" + connParams.getPort();
+ LOG.info("Resolved authority: " + authorityStr);
+ uri = uri.replace(dummyAuthorityString, authorityStr);
connParams.setJdbcUriString(uri);
- // Create a Java URI from the resolved URI for extracting the host/port
- URI resolvedAuthorityURI = null;
- try {
- resolvedAuthorityURI = new URI(null, resolvedAuthorityString, null, null, null);
- } catch (URISyntaxException e) {
- throw new JdbcUriParseException("Bad URL format: ", e);
- }
- connParams.setHost(resolvedAuthorityURI.getHost());
- connParams.setPort(resolvedAuthorityURI.getPort());
}
-
return connParams;
}
@@ -471,22 +461,17 @@ public class Utils {
return authorities;
}
- /**
- * Get a string representing a specific host:port
- * @param connParams
- * @return
- * @throws JdbcUriParseException
- * @throws ZooKeeperHiveClientException
- */
- private static String resolveAuthority(JdbcConnectionParams connParams)
+ private static void configureConnParams(JdbcConnectionParams connParams)
throws JdbcUriParseException, ZooKeeperHiveClientException {
String serviceDiscoveryMode =
connParams.getSessionVars().get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE);
if ((serviceDiscoveryMode != null)
&& (JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER
.equalsIgnoreCase(serviceDiscoveryMode))) {
- // Resolve using ZooKeeper
- return resolveAuthorityUsingZooKeeper(connParams);
+ // Set ZooKeeper ensemble in connParams for later use
+ connParams.setZooKeeperEnsemble(joinStringArray(connParams.getAuthorityList(), ","));
+ // Configure using ZooKeeper
+ ZooKeeperHiveClientHelper.configureConnParams(connParams);
} else {
String authority = connParams.getAuthorityList()[0];
URI jdbcURI = URI.create(URI_HIVE_PREFIX + "//" + authority);
@@ -494,32 +479,28 @@ public class Utils {
// to separate the 'path' portion of URI can result in this.
// The missing "/" common typo while using secure mode, eg of such url -
// jdbc:hive2://localhost:10000;principal=hive/HiveServer2Host@YOUR-REALM.COM
- if ((jdbcURI.getAuthority() != null) && (jdbcURI.getHost() == null)) {
- throw new JdbcUriParseException("Bad URL format. Hostname not found "
- + " in authority part of the url: " + jdbcURI.getAuthority()
- + ". Are you missing a '/' after the hostname ?");
+ if (jdbcURI.getAuthority() != null) {
+ String host = jdbcURI.getHost();
+ int port = jdbcURI.getPort();
+ if (host == null) {
+ throw new JdbcUriParseException("Bad URL format. Hostname not found "
+ + " in authority part of the url: " + jdbcURI.getAuthority()
+ + ". Are you missing a '/' after the hostname ?");
+ }
+ // Set the port to default value; we do support jdbc url like:
+ // jdbc:hive2://localhost/db
+ if (port <= 0) {
+ port = Integer.parseInt(Utils.DEFAULT_PORT);
+ }
+ connParams.setHost(jdbcURI.getHost());
+ connParams.setPort(jdbcURI.getPort());
}
- // Return the 1st element of the array
- return jdbcURI.getAuthority();
}
}
/**
- * Read a specific host:port from ZooKeeper
- * @param connParams
- * @return
- * @throws ZooKeeperHiveClientException
- */
- private static String resolveAuthorityUsingZooKeeper(JdbcConnectionParams connParams)
- throws ZooKeeperHiveClientException {
- // Set ZooKeeper ensemble in connParams for later use
- connParams.setZooKeeperEnsemble(joinStringArray(connParams.getAuthorityList(), ","));
- return ZooKeeperHiveClientHelper.getNextServerUriFromZooKeeper(connParams);
- }
-
- /**
* Read the next server coordinates (host:port combo) from ZooKeeper. Ignore the znodes already
- * explored. Also update the host, port, jdbcUriString fields of connParams.
+ * explored. Also update the host, port, jdbcUriString and other configs published by the server.
*
* @param connParams
* @throws ZooKeeperHiveClientException
@@ -528,25 +509,13 @@ public class Utils {
throws ZooKeeperHiveClientException {
// Add current host to the rejected list
connParams.getRejectedHostZnodePaths().add(connParams.getCurrentHostZnodePath());
- // Get another HiveServer2 uri from ZooKeeper
- String serverUriString = ZooKeeperHiveClientHelper.getNextServerUriFromZooKeeper(connParams);
- // Parse serverUri to a java URI and extract host, port
- URI serverUri = null;
- try {
- // Note URL_PREFIX is not a valid scheme format, therefore leaving it null in the constructor
- // to construct a valid URI
- serverUri = new URI(null, serverUriString, null, null, null);
- } catch (URISyntaxException e) {
- throw new ZooKeeperHiveClientException(e);
- }
String oldServerHost = connParams.getHost();
int oldServerPort = connParams.getPort();
- String newServerHost = serverUri.getHost();
- int newServerPort = serverUri.getPort();
- connParams.setHost(newServerHost);
- connParams.setPort(newServerPort);
+ // Update connection params (including host, port) from ZooKeeper
+ ZooKeeperHiveClientHelper.configureConnParams(connParams);
connParams.setJdbcUriString(connParams.getJdbcUriString().replace(
- oldServerHost + ":" + oldServerPort, newServerHost + ":" + newServerPort));
+ oldServerHost + ":" + oldServerPort, connParams.getHost() + ":" + connParams.getPort()));
+ LOG.info("Selected HiveServer2 instance with uri: " + connParams.getJdbcUriString());
}
private static String joinStringArray(String[] stringArray, String seperator) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
index e24b3dc..eeb3cf9 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
@@ -19,9 +19,10 @@
package org.apache.hive.jdbc;
import java.nio.charset.Charset;
-import java.sql.SQLException;
import java.util.List;
import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,26 +32,19 @@ import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
import org.apache.zookeeper.Watcher;
-public class ZooKeeperHiveClientHelper {
- public static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName());
-
+class ZooKeeperHiveClientHelper {
+ static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName());
+ // Pattern for key1=value1;key2=value2
+ private static final Pattern kvPattern = Pattern.compile("([^=;]*)=([^;]*)[;]?");
/**
* A no-op watcher class
*/
- public static class DummyWatcher implements Watcher {
+ static class DummyWatcher implements Watcher {
public void process(org.apache.zookeeper.WatchedEvent event) {
}
}
- /**
- * Resolve to a host:port by connecting to ZooKeeper and picking a host randomly.
- *
- * @param uri
- * @param connParams
- * @return
- * @throws SQLException
- */
- static String getNextServerUriFromZooKeeper(JdbcConnectionParams connParams)
+ static void configureConnParams(JdbcConnectionParams connParams)
throws ZooKeeperHiveClientException {
String zooKeeperEnsemble = connParams.getZooKeeperEnsemble();
String zooKeeperNamespace =
@@ -73,17 +67,17 @@ public class ZooKeeperHiveClientHelper {
throw new ZooKeeperHiveClientException(
"Tried all existing HiveServer2 uris from ZooKeeper.");
}
- // Now pick a host randomly
+ // Now pick a server node randomly
serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size()));
connParams.setCurrentHostZnodePath(serverNode);
- String serverUri =
+ // Read config string from the znode for this server node
+ String serverConfStr =
new String(
zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode),
Charset.forName("UTF-8"));
- LOG.info("Selected HiveServer2 instance with uri: " + serverUri);
- return serverUri;
+ applyConfs(serverConfStr, connParams);
} catch (Exception e) {
- throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper", e);
+ throw new ZooKeeperHiveClientException("Unable to read HiveServer2 configs from ZooKeeper", e);
} finally {
// Close the client connection with ZooKeeper
if (zooKeeperClient != null) {
@@ -91,4 +85,76 @@ public class ZooKeeperHiveClientHelper {
}
}
}
+
+ /**
+ * Apply configs published by the server. Configs specified from client's JDBC URI override
+ * configs published by the server.
+ *
+ * @param serverConfStr
+ * @param connParams
+ * @throws Exception
+ */
+ private static void applyConfs(String serverConfStr, JdbcConnectionParams connParams)
+ throws Exception {
+ Matcher matcher = kvPattern.matcher(serverConfStr);
+ while (matcher.find()) {
+ // Have to use this if-else since switch-case on String is supported Java 7 onwards
+ if ((matcher.group(1) != null)) {
+ if ((matcher.group(2) == null)) {
+ throw new Exception("Null config value for: " + matcher.group(1)
+ + " published by the server.");
+ }
+ // Set host
+ if ((matcher.group(1).equals("hive.server2.thrift.bind.host"))
+ && (connParams.getHost() == null)) {
+ connParams.setHost(matcher.group(2));
+ }
+ // Set transportMode
+ if ((matcher.group(1).equals("hive.server2.transport.mode"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.TRANSPORT_MODE))) {
+ connParams.getSessionVars().put(JdbcConnectionParams.TRANSPORT_MODE, matcher.group(2));
+ }
+ // Set port
+ if ((matcher.group(1).equals("hive.server2.thrift.port")) && !(connParams.getPort() > 0)) {
+ connParams.setPort(Integer.parseInt(matcher.group(2)));
+ }
+ if ((matcher.group(1).equals("hive.server2.thrift.http.port"))
+ && !(connParams.getPort() > 0)) {
+ connParams.setPort(Integer.parseInt(matcher.group(2)));
+ }
+ // Set sasl qop
+ if ((matcher.group(1).equals("hive.server2.thrift.sasl.qop"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_QOP))) {
+ connParams.getSessionVars().put(JdbcConnectionParams.AUTH_QOP, matcher.group(2));
+ }
+ // Set http path
+ if ((matcher.group(1).equals("hive.server2.thrift.http.path"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.HTTP_PATH))) {
+ connParams.getSessionVars().put(JdbcConnectionParams.HTTP_PATH, "/" + matcher.group(2));
+ }
+ // Set SSL
+ if ((matcher.group(1) != null) && (matcher.group(1).equals("hive.server2.use.SSL"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.USE_SSL))) {
+ connParams.getSessionVars().put(JdbcConnectionParams.USE_SSL, matcher.group(2));
+ }
+ // Set authentication configs
+ // Note that in JDBC driver, we have 3 auth modes: NOSASL, Kerberos and password based
+ // The use of "JdbcConnectionParams.AUTH_TYPE=JdbcConnectionParams.AUTH_SIMPLE" picks NOSASL
+ // The presence of "JdbcConnectionParams.AUTH_PRINCIPAL=<principal>" picks Kerberos
+ // Otherwise password based (which includes NONE, PAM, LDAP, CUSTOM)
+ if ((matcher.group(1).equals("hive.server2.authentication"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_TYPE))) {
+ if (matcher.group(2).equalsIgnoreCase("NOSASL")) {
+ connParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE,
+ JdbcConnectionParams.AUTH_SIMPLE);
+ }
+ }
+ // Set server's kerberos principal
+ if ((matcher.group(1).equals("hive.server2.authentication.kerberos.principal"))
+ && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_PRINCIPAL))) {
+ connParams.getSessionVars().put(JdbcConnectionParams.AUTH_PRINCIPAL, matcher.group(2));
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/service/src/java/org/apache/hive/service/server/HiveServer2.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 4a4be97..d7ba964 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -21,7 +21,9 @@ package org.apache.hive.service.server;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -69,6 +71,8 @@ import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooDefs.Perms;
import org.apache.zookeeper.data.ACL;
+import com.google.common.base.Joiner;
+
/**
* HiveServer2.
*
@@ -100,7 +104,12 @@ public class HiveServer2 extends CompositeService {
}
addService(thriftCLIService);
super.init(hiveConf);
-
+ // Set host name in hiveConf
+ try {
+ hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, getServerHost());
+ } catch (Throwable t) {
+ throw new Error("Unable to intitialize HiveServer2", t);
+ }
// Add a shutdown hook for catching SIGTERM & SIGINT
final HiveServer2 hiveServer2 = this;
Runtime.getRuntime().addShutdownHook(new Thread() {
@@ -122,6 +131,14 @@ public class HiveServer2 extends CompositeService {
return false;
}
+ public static boolean isKerberosAuthMode(HiveConf hiveConf) {
+ String authMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION);
+ if (authMode != null && (authMode.equalsIgnoreCase("KERBEROS"))) {
+ return true;
+ }
+ return false;
+ }
+
/**
* ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory
*/
@@ -158,9 +175,12 @@ public class HiveServer2 extends CompositeService {
private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception {
String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE);
- String instanceURI = getServerInstanceURI(hiveConf);
- byte[] znodeDataUTF8 = instanceURI.getBytes(Charset.forName("UTF-8"));
+ String instanceURI = getServerInstanceURI();
setUpZooKeeperAuth(hiveConf);
+ // HiveServer2 configs that this instance will publish to ZooKeeper,
+ // so that the clients can read these and configure themselves properly.
+ Map<String, String> confsToPublish = new HashMap<String, String>();
+ addConfsToPublish(hiveConf, confsToPublish);
int sessionTimeout =
(int) hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT,
TimeUnit.MILLISECONDS);
@@ -193,6 +213,10 @@ public class HiveServer2 extends CompositeService {
ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace
+ ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "serverUri=" + instanceURI + ";"
+ "version=" + HiveVersionInfo.getVersion() + ";" + "sequence=";
+ String znodeData = "";
+ // Publish configs for this instance as the data on the node
+ znodeData = Joiner.on(';').withKeyValueSeparator("=").join(confsToPublish);
+ byte[] znodeDataUTF8 = znodeData.getBytes(Charset.forName("UTF-8"));
znode =
new PersistentEphemeralNode(zooKeeperClient,
PersistentEphemeralNode.Mode.EPHEMERAL_SEQUENTIAL, pathPrefix, znodeDataUTF8);
@@ -220,6 +244,41 @@ public class HiveServer2 extends CompositeService {
}
/**
+ * Add conf keys, values that HiveServer2 will publish to ZooKeeper.
+ * @param hiveConf
+ */
+ private void addConfsToPublish(HiveConf hiveConf, Map<String, String> confsToPublish) {
+ // Hostname
+ confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST));
+ // Transport mode
+ confsToPublish.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE));
+ // Transport specific confs
+ if (isHTTPTransportMode(hiveConf)) {
+ confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT));
+ confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
+ } else {
+ confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP));
+ }
+ // Auth specific confs
+ confsToPublish.put(ConfVars.HIVE_SERVER2_AUTHENTICATION.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION));
+ if (isKerberosAuthMode(hiveConf)) {
+ confsToPublish.put(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL));
+ }
+ // SSL conf
+ confsToPublish.put(ConfVars.HIVE_SERVER2_USE_SSL.varname,
+ hiveConf.getVar(ConfVars.HIVE_SERVER2_USE_SSL));
+ }
+
+ /**
* For a kerberized cluster, we dynamically set up the client's JAAS conf.
*
* @param hiveConf
@@ -289,7 +348,7 @@ public class HiveServer2 extends CompositeService {
this.registeredWithZooKeeper = registeredWithZooKeeper;
}
- private String getServerInstanceURI(HiveConf hiveConf) throws Exception {
+ private String getServerInstanceURI() throws Exception {
if ((thriftCLIService == null) || (thriftCLIService.getServerIPAddress() == null)) {
throw new Exception("Unable to get the server address; it hasn't been initialized yet.");
}
@@ -297,6 +356,13 @@ public class HiveServer2 extends CompositeService {
+ thriftCLIService.getPortNumber();
}
+ private String getServerHost() throws Exception {
+ if ((thriftCLIService == null) || (thriftCLIService.getServerIPAddress() == null)) {
+ throw new Exception("Unable to get the server address; it hasn't been initialized yet.");
+ }
+ return thriftCLIService.getServerIPAddress().getHostName();
+ }
+
@Override
public synchronized void start() {
super.start();
[14/50] [abbrv] hive git commit: HIVE-11546: Projected columns read
size should be scaled to split size for ORC Splits (Prasanth Jayachandran
reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11546: Projected columns read size should be scaled to split size for ORC Splits (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8b2c605
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8b2c605
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8b2c605
Branch: refs/heads/llap
Commit: e8b2c605a05d06c1bfcd4c6bc611bc7f83306b38
Parents: cf0481f
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 14:21:51 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 14:21:51 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 30 ++++++-
.../apache/hadoop/hive/ql/io/orc/Reader.java | 7 ++
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 5 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 95 ++++++++++++++++++--
4 files changed, 125 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index fe2eccd..6ed7872 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -717,6 +717,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private ReaderImpl.FileMetaInfo fileMetaInfo;
private Metadata metadata;
private List<OrcProto.Type> types;
+ private boolean[] includedCols;
private final boolean isOriginal;
private final List<DeltaMetaData> deltas;
private final boolean hasBase;
@@ -830,8 +831,14 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
hosts = new String[hostList.size()];
hostList.toArray(hosts);
}
+
+ // scale the raw data size to split level based on ratio of split wrt to file length
+ final long fileLen = file.getLen();
+ final double splitRatio = (double) length / (double) fileLen;
+ final long scaledProjSize = projColsUncompressedSize > 0 ?
+ (long) (splitRatio * projColsUncompressedSize) : fileLen;
return new OrcSplit(file.getPath(), offset, length, hosts, fileMetaInfo,
- isOriginal, hasBase, deltas, projColsUncompressedSize);
+ isOriginal, hasBase, deltas, scaledProjSize);
}
/**
@@ -845,11 +852,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
// figure out which stripes we need to read
boolean[] includeStripe = null;
+
// we can't eliminate stripes if there are deltas because the
// deltas may change the rows making them match the predicate.
if (deltas.isEmpty()) {
Reader.Options options = new Reader.Options();
- options.include(genIncludedColumns(types, context.conf, isOriginal));
+ options.include(includedCols);
setSearchArgument(options, types, context.conf, isOriginal);
// only do split pruning if HIVE-8732 has been fixed in the writer
if (options.getSearchArgument() != null &&
@@ -930,8 +938,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private void populateAndCacheStripeDetails() throws IOException {
Reader orcReader = OrcFile.createReader(file.getPath(),
OrcFile.readerOptions(context.conf).filesystem(fs));
- List<String> projCols = ColumnProjectionUtils.getReadColumnNames(context.conf);
- // TODO: produce projColsUncompressedSize from projCols
if (fileInfo != null) {
stripes = fileInfo.stripeInfos;
fileMetaInfo = fileInfo.fileMetaInfo;
@@ -959,6 +965,22 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
metadata, types, fileMetaInfo, writerVersion));
}
}
+ includedCols = genIncludedColumns(types, context.conf, isOriginal);
+ projColsUncompressedSize = computeProjectionSize(orcReader, includedCols, isOriginal);
+ }
+
+ private long computeProjectionSize(final Reader orcReader, final boolean[] includedCols,
+ final boolean isOriginal) {
+ final int rootIdx = getRootColumn(isOriginal);
+ List<Integer> internalColIds = Lists.newArrayList();
+ if (includedCols != null) {
+ for (int i = 0; i < includedCols.length; i++) {
+ if (includedCols[i]) {
+ internalColIds.add(rootIdx + i);
+ }
+ }
+ }
+ return orcReader.getRawDataSizeFromColIndices(internalColIds);
}
private boolean isStripeSatisfyPredicate(StripeStatistics stripeStatistics,
http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
index 6f4f013..7bddefc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
@@ -52,6 +52,13 @@ public interface Reader {
long getRawDataSizeOfColumns(List<String> colNames);
/**
+ * Get the deserialized data size of the specified columns ids
+ * @param colIds - internal column id (check orcfiledump for column ids)
+ * @return raw data size of columns
+ */
+ long getRawDataSizeFromColIndices(List<Integer> colIds);
+
+ /**
* Get the user metadata keys.
* @return the set of metadata keys
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 3c0de3c..a6448b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -575,7 +575,8 @@ public class ReaderImpl implements Reader {
return deserializedSize;
}
- private long getRawDataSizeFromColIndices(List<Integer> colIndices) {
+ @Override
+ public long getRawDataSizeFromColIndices(List<Integer> colIndices) {
long result = 0;
for (int colIdx : colIndices) {
result += getRawDataSizeOfColumn(colIdx);
@@ -620,7 +621,7 @@ public class ReaderImpl implements Reader {
case BYTE:
return numVals * JavaDataModel.get().primitive1();
default:
- LOG.debug("Unknown primitive category.");
+ LOG.debug("Unknown primitive category: " + type.getKind());
break;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 6cb8529..0c12c89 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Output;
-
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -108,6 +105,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Output;
+
public class TestInputOutputFormat {
public static String toKryo(SearchArgument sarg) {
@@ -902,14 +902,25 @@ public class TestInputOutputFormat {
}
fill(buffer, offset);
footer.addTypes(OrcProto.Type.newBuilder()
- .setKind(OrcProto.Type.Kind.STRUCT)
- .addFieldNames("col1")
- .addSubtypes(1));
+ .setKind(OrcProto.Type.Kind.STRUCT)
+ .addFieldNames("col1")
+ .addSubtypes(1));
footer.addTypes(OrcProto.Type.newBuilder()
.setKind(OrcProto.Type.Kind.STRING));
footer.setNumberOfRows(1000 * stripeLengths.length)
.setHeaderLength(headerLen)
.setContentLength(offset - headerLen);
+ footer.addStatistics(OrcProto.ColumnStatistics.newBuilder()
+ .setNumberOfValues(1000 * stripeLengths.length).build());
+ footer.addStatistics(OrcProto.ColumnStatistics.newBuilder()
+ .setNumberOfValues(1000 * stripeLengths.length)
+ .setStringStatistics(
+ OrcProto.StringStatistics.newBuilder()
+ .setMaximum("zzz")
+ .setMinimum("aaa")
+ .setSum(1000 * 3 * stripeLengths.length)
+ .build()
+ ).build());
footer.build().writeTo(buffer);
int footerEnd = buffer.getLength();
OrcProto.PostScript ps =
@@ -1013,6 +1024,78 @@ public class TestInputOutputFormat {
}
@Test
+ public void testProjectedColumnSize() throws Exception {
+ long[] stripeSizes =
+ new long[]{200, 200, 200, 200, 100};
+ MockFileSystem fs = new MockFileSystem(conf,
+ new MockFile("mock:/a/file", 500,
+ createMockOrcFile(stripeSizes),
+ new MockBlock("host1-1", "host1-2", "host1-3"),
+ new MockBlock("host2-1", "host0", "host2-3"),
+ new MockBlock("host0", "host3-2", "host3-3"),
+ new MockBlock("host4-1", "host4-2", "host4-3"),
+ new MockBlock("host5-1", "host5-2", "host5-3")));
+ conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300);
+ conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200);
+ conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
+ conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0");
+ OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
+ OrcInputFormat.SplitGenerator splitter =
+ new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
+ fs.getFileStatus(new Path("/a/file")), null, true,
+ new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null));
+ List<OrcSplit> results = splitter.call();
+ OrcSplit result = results.get(0);
+ assertEquals(3, results.size());
+ assertEquals(3, result.getStart());
+ assertEquals(400, result.getLength());
+ assertEquals(167468, result.getProjectedColumnsUncompressedSize());
+ result = results.get(1);
+ assertEquals(403, result.getStart());
+ assertEquals(400, result.getLength());
+ assertEquals(167468, result.getProjectedColumnsUncompressedSize());
+ result = results.get(2);
+ assertEquals(803, result.getStart());
+ assertEquals(100, result.getLength());
+ assertEquals(41867, result.getProjectedColumnsUncompressedSize());
+
+ // test min = 0, max = 0 generates each stripe
+ conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0);
+ conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
+ context = new OrcInputFormat.Context(conf);
+ splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
+ fs.getFileStatus(new Path("/a/file")), null, true,
+ new ArrayList<AcidInputFormat.DeltaMetaData>(),
+ true, null, null));
+ results = splitter.call();
+ assertEquals(5, results.size());
+ for (int i = 0; i < stripeSizes.length; ++i) {
+ assertEquals("checking stripe " + i + " size",
+ stripeSizes[i], results.get(i).getLength());
+ if (i == stripeSizes.length - 1) {
+ assertEquals(41867, results.get(i).getProjectedColumnsUncompressedSize());
+ } else {
+ assertEquals(83734, results.get(i).getProjectedColumnsUncompressedSize());
+ }
+ }
+
+ // single split
+ conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 100000);
+ conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 1000);
+ context = new OrcInputFormat.Context(conf);
+ splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
+ fs.getFileStatus(new Path("/a/file")), null, true,
+ new ArrayList<AcidInputFormat.DeltaMetaData>(),
+ true, null, null));
+ results = splitter.call();
+ assertEquals(1, results.size());
+ result = results.get(0);
+ assertEquals(3, result.getStart());
+ assertEquals(900, result.getLength());
+ assertEquals(376804, result.getProjectedColumnsUncompressedSize());
+ }
+
+ @Test
@SuppressWarnings("unchecked,deprecation")
public void testInOutFormat() throws Exception {
Properties properties = new Properties();
[03/50] [abbrv] hive git commit: HIVE-11464: lineage info missing if
there are multiple outputs (Jimmy)
Posted by se...@apache.org.
HIVE-11464: lineage info missing if there are multiple outputs (Jimmy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a75644d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a75644d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a75644d
Branch: refs/heads/llap
Commit: 1a75644d68c8c61fbafb4058fe45b7823492491c
Parents: f26b256
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Wed Aug 5 08:02:50 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Thu Aug 13 13:44:03 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 8 ++--
.../hadoop/hive/ql/hooks/LineageInfo.java | 9 ++--
.../hadoop/hive/ql/hooks/LineageLogger.java | 44 +++++++++++++-------
.../ql/optimizer/lineage/ExprProcFactory.java | 9 ++--
.../hive/ql/optimizer/lineage/LineageCtx.java | 34 +++++++++++----
.../ql/optimizer/lineage/OpProcFactory.java | 10 ++---
ql/src/test/queries/clientpositive/lineage3.q | 15 +++++++
.../test/results/clientpositive/lineage3.q.out | 32 +++++++++++++-
8 files changed, 118 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index e7b7b55..c0c1b2e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -441,8 +441,11 @@ public class Driver implements CommandProcessor {
// to avoid returning sensitive data
String queryStr = HookUtils.redactLogString(conf, command);
+ // get the output schema
+ schema = getSchema(sem, conf);
+
plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
- SessionState.get().getHiveOperation(), getSchema(sem, conf));
+ SessionState.get().getHiveOperation(), schema);
conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);
@@ -454,9 +457,6 @@ public class Driver implements CommandProcessor {
plan.getFetchTask().initialize(conf, plan, null);
}
- // get the output schema
- schema = getSchema(sem, conf);
-
//do the authorization check
if (!sem.skipAuthorization() &&
HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
index fe0841e..2806c54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
@@ -22,7 +22,6 @@ import java.io.Serializable;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -375,9 +374,9 @@ public class LineageInfo implements Serializable {
private String expr;
/**
- * The list of base columns that the particular column depends on.
+ * The set of base columns that the particular column depends on.
*/
- private List<BaseColumnInfo> baseCols;
+ private Set<BaseColumnInfo> baseCols;
/**
* @return the type
@@ -410,14 +409,14 @@ public class LineageInfo implements Serializable {
/**
* @return the baseCols
*/
- public List<BaseColumnInfo> getBaseCols() {
+ public Set<BaseColumnInfo> getBaseCols() {
return baseCols;
}
/**
* @param baseCols the baseCols to set
*/
- public void setBaseCols(List<BaseColumnInfo> baseCols) {
+ public void setBaseCols(Set<BaseColumnInfo> baseCols) {
this.baseCols = baseCols;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
index d615372..3c6ce94 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
@@ -33,6 +33,7 @@ import org.apache.commons.io.output.StringBuilderWriter;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -147,6 +148,7 @@ public class LineageLogger implements ExecuteWithHookContext {
// Don't emit user/timestamp info in test mode,
// so that the test golden output file is fixed.
long queryTime = plan.getQueryStartTime().longValue();
+ if (queryTime == 0) queryTime = System.currentTimeMillis();
writer.name("user").value(hookContext.getUgi().getUserName());
writer.name("timestamp").value(queryTime/1000);
writer.name("jobIds");
@@ -209,23 +211,28 @@ public class LineageLogger implements ExecuteWithHookContext {
* For each target column, find out its sources based on the dependency index.
*/
private List<Edge> getEdges(QueryPlan plan, Index index) {
- List<FieldSchema> fieldSchemas = plan.getResultSchema().getFieldSchemas();
- int fields = fieldSchemas == null ? 0 : fieldSchemas.size();
- SelectOperator finalSelOp = index.getFinalSelectOp();
+ LinkedHashMap<String, ObjectPair<SelectOperator,
+ org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = index.getFinalSelectOps();
+ Set<Vertex> allTargets = new LinkedHashSet<Vertex>();
+ Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>();
List<Edge> edges = new ArrayList<Edge>();
- if (finalSelOp != null && fields > 0) {
- Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp);
- List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null;
- if (dependencies == null || dependencies.size() != fields) {
- log("Result schema has " + fields
- + " fields, but we don't get as many dependencies");
+ for (ObjectPair<SelectOperator,
+ org.apache.hadoop.hive.ql.metadata.Table> pair: finalSelOps.values()) {
+ List<FieldSchema> fieldSchemas = plan.getResultSchema().getFieldSchemas();
+ SelectOperator finalSelOp = pair.getFirst();
+ org.apache.hadoop.hive.ql.metadata.Table t = pair.getSecond();
+ String destTableName = null;
+ List<String> colNames = null;
+ if (t != null) {
+ destTableName = t.getDbName() + "." + t.getTableName();
+ fieldSchemas = t.getCols();
} else {
- String destTableName = null;
- List<String> colNames = null;
// Based on the plan outputs, find out the target table name and column names.
for (WriteEntity output : plan.getOutputs()) {
- if (output.getType() == Entity.Type.TABLE) {
- org.apache.hadoop.hive.ql.metadata.Table t = output.getTable();
+ Entity.Type entityType = output.getType();
+ if (entityType == Entity.Type.TABLE
+ || entityType == Entity.Type.PARTITION) {
+ t = output.getTable();
destTableName = t.getDbName() + "." + t.getTableName();
List<FieldSchema> cols = t.getCols();
if (cols != null && !cols.isEmpty()) {
@@ -234,10 +241,15 @@ public class LineageLogger implements ExecuteWithHookContext {
break;
}
}
-
+ }
+ int fields = fieldSchemas.size();
+ Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp);
+ List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null;
+ if (dependencies == null || dependencies.size() != fields) {
+ log("Result schema has " + fields
+ + " fields, but we don't get as many dependencies");
+ } else {
// Go through each target column, generate the lineage edges.
- Set<Vertex> allTargets = new LinkedHashSet<Vertex>();
- Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>();
for (int i = 0; i < fields; i++) {
Vertex target = new Vertex(
getTargetFieldName(i, destTableName, colNames, fieldSchemas));
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
index 455a525..38040e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
@@ -24,6 +24,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Stack;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -124,7 +125,7 @@ public class ExprProcFactory {
bci_set.addAll(child_dep.getBaseCols());
}
- dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
+ dep.setBaseCols(bci_set);
dep.setType(new_type);
return dep;
@@ -146,7 +147,7 @@ public class ExprProcFactory {
// Create a dependency that has no basecols
Dependency dep = new Dependency();
dep.setType(LineageInfo.DependencyType.SIMPLE);
- dep.setBaseCols(new ArrayList<BaseColumnInfo>());
+ dep.setBaseCols(new LinkedHashSet<BaseColumnInfo>());
return dep;
}
@@ -218,9 +219,9 @@ public class ExprProcFactory {
Dependency dep = lctx.getIndex().getDependency(inpOp, internalName);
if ((tabAlias == null || tabAlias.startsWith("_") || tabAlias.startsWith("$"))
&& (dep != null && dep.getType() == DependencyType.SIMPLE)) {
- List<BaseColumnInfo> baseCols = dep.getBaseCols();
+ Set<BaseColumnInfo> baseCols = dep.getBaseCols();
if (baseCols != null && !baseCols.isEmpty()) {
- BaseColumnInfo baseCol = baseCols.get(0);
+ BaseColumnInfo baseCol = baseCols.iterator().next();
tabAlias = baseCol.getTabAlias().getAlias();
alias = baseCol.getColumn().getName();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
index d26d8da..c33d775 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
@@ -25,7 +25,9 @@ import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
@@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo.BaseColumnInfo;
import org.apache.hadoop.hive.ql.hooks.LineageInfo.Dependency;
import org.apache.hadoop.hive.ql.hooks.LineageInfo.Predicate;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -59,7 +62,11 @@ public class LineageCtx implements NodeProcessorCtx {
*/
private final Map<Operator<? extends OperatorDesc>, Set<Predicate>> condMap;
- private SelectOperator finalSelectOp;
+ /**
+ * A map from a final select operator id to the select operator
+ * and the corresponding target table in case an insert into query.
+ */
+ private LinkedHashMap<String, ObjectPair<SelectOperator, Table>> finalSelectOps;
/**
* Constructor.
@@ -69,6 +76,8 @@ public class LineageCtx implements NodeProcessorCtx {
new LinkedHashMap<Operator<? extends OperatorDesc>,
LinkedHashMap<ColumnInfo, Dependency>>();
condMap = new HashMap<Operator<? extends OperatorDesc>, Set<Predicate>>();
+ finalSelectOps =
+ new LinkedHashMap<String, ObjectPair<SelectOperator, Table>>();
}
/**
@@ -146,7 +155,7 @@ public class LineageCtx implements NodeProcessorCtx {
old_dep.setType(new_type);
Set<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(old_dep.getBaseCols());
bci_set.addAll(dep.getBaseCols());
- old_dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
+ old_dep.setBaseCols(bci_set);
// TODO: Fix the expressions later.
old_dep.setExpr(null);
}
@@ -179,16 +188,27 @@ public class LineageCtx implements NodeProcessorCtx {
return condMap.get(op);
}
- public void setFinalSelectOp(SelectOperator sop) {
- finalSelectOp = sop;
+ public void addFinalSelectOp(
+ SelectOperator sop, Operator<? extends OperatorDesc> sinkOp) {
+ String operatorId = sop.getOperatorId();
+ if (!finalSelectOps.containsKey(operatorId)) {
+ Table table = null;
+ if (sinkOp instanceof FileSinkOperator) {
+ FileSinkOperator fso = (FileSinkOperator) sinkOp;
+ table = fso.getConf().getTable();
+ }
+ finalSelectOps.put(operatorId,
+ new ObjectPair<SelectOperator, Table>(sop, table));
+ }
}
- public SelectOperator getFinalSelectOp() {
- return finalSelectOp;
+ public LinkedHashMap<String,
+ ObjectPair<SelectOperator, Table>> getFinalSelectOps() {
+ return finalSelectOps;
}
public void clear() {
- finalSelectOp = null;
+ finalSelectOps.clear();
depMap.clear();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
index f670db8..5c5d0d6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
@@ -120,7 +120,7 @@ public class OpProcFactory {
}
dep.setType(new_type);
- dep.setBaseCols(new ArrayList<BaseColumnInfo>(col_set));
+ dep.setBaseCols(col_set);
boolean isScript = op instanceof ScriptOperator;
@@ -186,7 +186,7 @@ public class OpProcFactory {
// Populate the dependency
dep.setType(LineageInfo.DependencyType.SIMPLE);
- dep.setBaseCols(new ArrayList<BaseColumnInfo>());
+ dep.setBaseCols(new LinkedHashSet<BaseColumnInfo>());
dep.getBaseCols().add(bci);
// Put the dependency in the map
@@ -396,7 +396,7 @@ public class OpProcFactory {
}
if (op == null || (op.getChildOperators().isEmpty()
&& op instanceof FileSinkOperator)) {
- lctx.getIndex().setFinalSelectOp(sop);
+ lctx.getIndex().addFinalSelectOp(sop, op);
}
return null;
@@ -450,7 +450,7 @@ public class OpProcFactory {
new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type);
bci_set.addAll(expr_dep.getBaseCols());
if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) {
- BaseColumnInfo col = expr_dep.getBaseCols().get(0);
+ BaseColumnInfo col = expr_dep.getBaseCols().iterator().next();
Table t = col.getTabAlias().getTable();
if (t != null) {
sb.append(t.getDbName()).append(".").append(t.getTableName()).append(".");
@@ -514,7 +514,7 @@ public class OpProcFactory {
}
}
- dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
+ dep.setBaseCols(bci_set);
dep.setType(new_type);
lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/queries/clientpositive/lineage3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q
index 53fff0f..c24ff7d 100644
--- a/ql/src/test/queries/clientpositive/lineage3.q
+++ b/ql/src/test/queries/clientpositive/lineage3.q
@@ -1,5 +1,20 @@
set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.LineageLogger;
+drop table if exists d1;
+create table d1(a int);
+
+from (select a.ctinyint x, b.cstring1 y
+from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
+insert into table d1 select x + length(y);
+
+drop table if exists d2;
+create table d2(b varchar(128));
+
+from (select a.ctinyint x, b.cstring1 y
+from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
+insert into table d1 select x where y is null
+insert into table d2 select y where x > 0;
+
drop table if exists t;
create table t as
select * from
http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/results/clientpositive/lineage3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out
index 75d88f8..b6b4e0b 100644
--- a/ql/src/test/results/clientpositive/lineage3.q.out
+++ b/ql/src/test/results/clientpositive/lineage3.q.out
@@ -1,3 +1,31 @@
+PREHOOK: query: drop table if exists d1
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table d1(a int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@d1
+PREHOOK: query: from (select a.ctinyint x, b.cstring1 y
+from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
+insert into table d1 select x + length(y)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@d1
+{"version":"1.0","engine":"mr","hash":"4c9b7b8d89403cef78668f15d393e542","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x + length(y)","edges":[{"sources":[1,2],"targets":[0],"expression":"(UDFToInteger(a.ctinyint) + length(a.cstring1))","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0],"expression":"(UDFToLong(a.cint) = a.cbigint)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
+PREHOOK: query: drop table if exists d2
+PREHOOK: type: DROPTABLE
+PREHOOK: query: create table d2(b varchar(128))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@d2
+PREHOOK: query: from (select a.ctinyint x, b.cstring1 y
+from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
+insert into table d1 select x where y is null
+insert into table d2 select y where x > 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: default@d1
+PREHOOK: Output: default@d2
+{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[4,5],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[3],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.
cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
PREHOOK: query: drop table if exists t
PREHOOK: type: DROPTABLE
PREHOOK: query: create table t as
@@ -23,7 +51,7 @@ where cint is not null and cint < 0 order by cint, cs limit 5
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: default@dest_l1@ds=today
-{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"cint"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
+{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
PREHOOK: query: insert into table dest_l1 partition (ds='tomorrow')
select min(cint), cast(min(cstring1) as varchar(128)) as cs
from alltypesorc
@@ -33,7 +61,7 @@ having min(cbigint) > 10
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: default@dest_l1@ds=tomorrow
-{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN",
"vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
+{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},
{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
PREHOOK: query: select cint, rank() over(order by cint) from alltypesorc
where cint > 10 and cint < 10000 limit 10
PREHOOK: type: QUERY
[05/50] [abbrv] hive git commit: HIVE-11538 : Add an option to skip
init script while running tests (Ashutosh Chauhan via Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11538 : Add an option to skip init script while running tests (Ashutosh Chauhan via Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e762919
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e762919
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e762919
Branch: refs/heads/llap
Commit: 6e7629193ed7b8714bd5ae5ab48bd10c2cbd85cf
Parents: 433ea9c
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Thu Aug 13 20:42:33 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Aug 13 20:43:46 2015 -0700
----------------------------------------------------------------------
itests/qtest/pom.xml | 26 ++++----
.../org/apache/hadoop/hive/ql/QTestUtil.java | 62 ++++++++++----------
2 files changed, 44 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 44d30da..0588233 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -30,7 +30,7 @@
<properties>
<hive.path.to.root>../..</hive.path.to.root>
-
+ <initScript>q_test_init.sql</initScript>
<qfile></qfile>
<qfile_regex></qfile_regex>
<run_disabled>false</run_disabled>
@@ -420,7 +420,7 @@
logFile="${project.build.directory}/testparseneggen.log"
hadoopVersion="${active.hadoop.version}"
logDirectory="${project.build.directory}/qfile-results/negative/"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- Cli -->
@@ -437,7 +437,7 @@
logFile="${project.build.directory}/testclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- Negative Cli -->
@@ -454,7 +454,7 @@
logFile="${project.build.directory}/testnegativeclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
hadoopVersion="${active.hadoop.version}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- Compare Cli -->
@@ -470,7 +470,7 @@
logFile="${project.build.directory}/testcompareclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientcompare/"
hadoopVersion="${active.hadoop.version}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- Minimr -->
@@ -487,7 +487,7 @@
logFile="${project.build.directory}/testminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<if>
@@ -508,7 +508,7 @@
logFile="${project.build.directory}/testminitezclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/"
hadoopVersion="${active.hadoop.version}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -547,7 +547,7 @@
logFile="${project.build.directory}/testnegativeminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientnegative/"
hadoopVersion="${hadoopVersion}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Positive -->
@@ -561,7 +561,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseCliDriver"
logFile="${project.build.directory}/testhbaseclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Minimr -->
@@ -575,7 +575,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseMinimrCliDriver"
logFile="${project.build.directory}/testhbaseminimrclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- HBase Negative -->
@@ -589,7 +589,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/negative/" className="TestHBaseNegativeCliDriver"
logFile="${project.build.directory}/testhbasenegativeclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<!-- Beeline -->
@@ -626,7 +626,7 @@
logFile="${project.build.directory}/testcontribclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/contribclientpositive"
hadoopVersion="${hadoopVersion}"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
@@ -639,7 +639,7 @@
resultsDirectory="${basedir}/${hive.path.to.root}/contrib/src/test/results/clientnegative/" className="TestContribNegativeCliDriver"
logFile="${project.build.directory}/testcontribnegclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/contribclientnegative"
- initScript="q_test_init.sql"
+ initScript="${initScript}"
cleanupScript="q_test_cleanup.sql"/>
http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 39d5d9e..3fae0ba 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -121,8 +121,8 @@ public class QTestUtil {
private static final Log LOG = LogFactory.getLog("QTestUtil");
private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
- private final String defaultInitScript = "q_test_init.sql";
- private final String defaultCleanupScript = "q_test_cleanup.sql";
+ private final static String defaultInitScript = "q_test_init.sql";
+ private final static String defaultCleanupScript = "q_test_cleanup.sql";
private final String[] testOnlyCommands = new String[]{"crypto"};
private String testWarehouse;
@@ -149,7 +149,6 @@ public class QTestUtil {
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
private HadoopShims.HdfsEncryptionShim hes = null;
- private boolean miniMr = false;
private String hadoopVer = null;
private QTestSetup setup = null;
private SparkSession sparkSession = null;
@@ -209,7 +208,7 @@ public class QTestUtil {
continue;
}
- if (file.isDir()) {
+ if (file.isDirectory()) {
if (!destFs.exists(local_path)) {
destFs.mkdirs(local_path);
}
@@ -410,14 +409,9 @@ public class QTestUtil {
if (scriptsDir == null) {
scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
}
- if (initScript.isEmpty()) {
- initScript = defaultInitScript;
- }
- if (cleanupScript.isEmpty()) {
- cleanupScript = defaultCleanupScript;
- }
- this.initScript = scriptsDir + "/" + initScript;
- this.cleanupScript = scriptsDir + "/" + cleanupScript;
+
+ this.initScript = scriptsDir + File.separator + initScript;
+ this.cleanupScript = scriptsDir + File.separator + cleanupScript;
overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
@@ -705,7 +699,7 @@ public class QTestUtil {
FileSystem fileSystem = p.getFileSystem(conf);
if (fileSystem.exists(p)) {
for (FileStatus status : fileSystem.listStatus(p)) {
- if (status.isDir() && !srcTables.contains(status.getPath().getName())) {
+ if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) {
fileSystem.delete(status.getPath(), true);
}
}
@@ -755,16 +749,19 @@ public class QTestUtil {
clearTablesCreatedDuringTests();
clearKeysCreatedInTests();
- SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
-
- String cleanupCommands = readEntireFileIntoString(new File(cleanupScript));
- LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
- if(cliDriver == null) {
- cliDriver = new CliDriver();
+ File cleanupFile = new File(cleanupScript);
+ if (cleanupFile.isFile()) {
+ String cleanupCommands = readEntireFileIntoString(cleanupFile);
+ LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
+ cliDriver.processLine(cleanupCommands);
+ SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
+ } else {
+ LOG.info("No cleanup script detected. Skipping.");
}
- cliDriver.processLine(cleanupCommands);
-
- SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
// delete any contents in the warehouse dir
Path p = new Path(testWarehouse);
@@ -809,14 +806,21 @@ public class QTestUtil {
if(!isSessionStateStarted) {
startSessionState();
}
- conf.setBoolean("hive.test.init.phase", true);
- String initCommands = readEntireFileIntoString(new File(this.initScript));
- LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
if(cliDriver == null) {
cliDriver = new CliDriver();
}
cliDriver.processLine("set test.data.dir=" + testFiles + ";");
+ File scriptFile = new File(this.initScript);
+ if (!scriptFile.isFile()) {
+ LOG.info("No init script detected. Skipping");
+ return;
+ }
+ conf.setBoolean("hive.test.init.phase", true);
+
+ String initCommands = readEntireFileIntoString(scriptFile);
+ LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
+
cliDriver.processLine(initCommands);
conf.setBoolean("hive.test.init.phase", false);
@@ -912,6 +916,7 @@ public class QTestUtil {
private CliSessionState createSessionState() {
return new CliSessionState(conf) {
+ @Override
public void setSparkSession(SparkSession sparkSession) {
super.setSparkSession(sparkSession);
if (sparkSession != null) {
@@ -1136,11 +1141,6 @@ public class QTestUtil {
return commands;
}
- private boolean isComment(final String line) {
- String lineTrimmed = line.trim();
- return lineTrimmed.startsWith("#") || lineTrimmed.startsWith("--");
- }
-
public boolean shouldBeSkipped(String tname) {
return qSkipSet.contains(tname);
}
@@ -1816,7 +1816,7 @@ public class QTestUtil {
{
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
- qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", "", "");
+ qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", defaultInitScript, defaultCleanupScript);
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
[24/50] [abbrv] hive git commit: HIVE-11542 : port fileId support on
shims and splits from llap branch (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-11542 : port fileId support on shims and splits from llap branch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3b6825b5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3b6825b5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3b6825b5
Branch: refs/heads/llap
Commit: 3b6825b5b61e943e8e41743f5cbf6d640e0ebdf5
Parents: e059409
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Aug 17 15:16:57 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Aug 17 15:16:57 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 183 ++++++++++++++-----
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 97 +++++++---
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 25 ++-
.../hive/ql/txn/compactor/CompactorMR.java | 13 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 9 +-
.../apache/hadoop/hive/ql/io/TestAcidUtils.java | 27 +--
.../hive/ql/io/orc/TestInputOutputFormat.java | 6 +-
.../hadoop/hive/shims/Hadoop20SShims.java | 11 ++
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 66 +++++++
.../apache/hadoop/hive/shims/HadoopShims.java | 15 ++
11 files changed, 348 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9a6781b..da171b1 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1024,6 +1024,8 @@ public class HiveConf extends Configuration {
"data is read remotely (from the client or HS2 machine) and sent to all the tasks."),
HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE("hive.orc.cache.stripe.details.size", 10000,
"Max cache size for keeping meta info about orc splits cached in the client."),
+ HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS("hive.orc.splits.include.fileid", true,
+ "Include file ID in splits on file systems thaty support it."),
HIVE_ORC_COMPUTE_SPLITS_NUM_THREADS("hive.orc.compute.splits.num.threads", 10,
"How many threads orc should use to create splits in parallel."),
HIVE_ORC_SKIP_CORRUPT_DATA("hive.exec.orc.skip.corrupt.data", false,
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index c7e0780..30db513 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -28,6 +28,9 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
+
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.ArrayList;
@@ -221,7 +224,7 @@ public class AcidUtils {
* Get the list of original files.
* @return the list of original files (eg. 000000_0)
*/
- List<FileStatus> getOriginalFiles();
+ List<HdfsFileStatusWithId> getOriginalFiles();
/**
* Get the list of base and delta directories that are valid and not
@@ -423,6 +426,20 @@ public class AcidUtils {
return false;
}
+ @VisibleForTesting
+ public static Directory getAcidState(Path directory,
+ Configuration conf,
+ ValidTxnList txnList
+ ) throws IOException {
+ return getAcidState(directory, conf, txnList, false);
+ }
+
+ /** State class for getChildState; cannot modify 2 things in a method. */
+ private static class TxnBase {
+ private FileStatus status;
+ private long txn;
+ }
+
/**
* Get the ACID state of the given directory. It finds the minimal set of
* base and diff directories. Note that because major compactions don't
@@ -436,51 +453,40 @@ public class AcidUtils {
*/
public static Directory getAcidState(Path directory,
Configuration conf,
- ValidTxnList txnList
+ ValidTxnList txnList,
+ boolean useFileIds
) throws IOException {
FileSystem fs = directory.getFileSystem(conf);
- FileStatus bestBase = null;
- long bestBaseTxn = 0;
final List<ParsedDelta> deltas = new ArrayList<ParsedDelta>();
List<ParsedDelta> working = new ArrayList<ParsedDelta>();
List<FileStatus> originalDirectories = new ArrayList<FileStatus>();
final List<FileStatus> obsolete = new ArrayList<FileStatus>();
- List<FileStatus> children = SHIMS.listLocatedStatus(fs, directory,
- hiddenFileFilter);
- for(FileStatus child: children) {
- Path p = child.getPath();
- String fn = p.getName();
- if (fn.startsWith(BASE_PREFIX) && child.isDir()) {
- long txn = parseBase(p);
- if (bestBase == null) {
- bestBase = child;
- bestBaseTxn = txn;
- } else if (bestBaseTxn < txn) {
- obsolete.add(bestBase);
- bestBase = child;
- bestBaseTxn = txn;
- } else {
- obsolete.add(child);
- }
- } else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) {
- ParsedDelta delta = parseDelta(child);
- if (txnList.isTxnRangeValid(delta.minTransaction,
- delta.maxTransaction) !=
- ValidTxnList.RangeResponse.NONE) {
- working.add(delta);
- }
- } else {
- // This is just the directory. We need to recurse and find the actual files. But don't
- // do this until we have determined there is no base. This saves time. Plus,
- // it is possible that the cleaner is running and removing these original files,
- // in which case recursing through them could cause us to get an error.
- originalDirectories.add(child);
+ List<HdfsFileStatusWithId> childrenWithId = null;
+ if (useFileIds) {
+ try {
+ childrenWithId = SHIMS.listLocatedHdfsStatus(fs, directory, hiddenFileFilter);
+ } catch (Throwable t) {
+ LOG.error("Failed to get files with ID; using regular API", t);
+ useFileIds = false;
+ }
+ }
+ TxnBase bestBase = new TxnBase();
+ final List<HdfsFileStatusWithId> original = new ArrayList<>();
+ if (childrenWithId != null) {
+ for (HdfsFileStatusWithId child : childrenWithId) {
+ getChildState(child.getFileStatus(), child, txnList, working,
+ originalDirectories, original, obsolete, bestBase);
+ }
+ } else {
+ List<FileStatus> children = SHIMS.listLocatedStatus(fs, directory, hiddenFileFilter);
+ for (FileStatus child : children) {
+ getChildState(
+ child, null, txnList, working, originalDirectories, original, obsolete, bestBase);
}
}
- final List<FileStatus> original = new ArrayList<FileStatus>();
- // if we have a base, the original files are obsolete.
- if (bestBase != null) {
+ // If we have a base, the original files are obsolete.
+ if (bestBase.status != null) {
// remove the entries so we don't get confused later and think we should
// use them.
original.clear();
@@ -488,12 +494,12 @@ public class AcidUtils {
// Okay, we're going to need these originals. Recurse through them and figure out what we
// really need.
for (FileStatus origDir : originalDirectories) {
- findOriginals(fs, origDir, original);
+ findOriginals(fs, origDir, original, useFileIds);
}
}
Collections.sort(working);
- long current = bestBaseTxn;
+ long current = bestBase.txn;
int lastStmtId = -1;
for(ParsedDelta next: working) {
if (next.maxTransaction > current) {
@@ -516,7 +522,7 @@ public class AcidUtils {
}
}
- final Path base = bestBase == null ? null : bestBase.getPath();
+ final Path base = bestBase.status == null ? null : bestBase.status.getPath();
LOG.debug("in directory " + directory.toUri().toString() + " base = " + base + " deltas = " +
deltas.size());
@@ -528,7 +534,7 @@ public class AcidUtils {
}
@Override
- public List<FileStatus> getOriginalFiles() {
+ public List<HdfsFileStatusWithId> getOriginalFiles() {
return original;
}
@@ -544,23 +550,100 @@ public class AcidUtils {
};
}
+ private static void getChildState(FileStatus child, HdfsFileStatusWithId childWithId,
+ ValidTxnList txnList, List<ParsedDelta> working, List<FileStatus> originalDirectories,
+ List<HdfsFileStatusWithId> original, List<FileStatus> obsolete, TxnBase bestBase) {
+ Path p = child.getPath();
+ String fn = p.getName();
+ if (fn.startsWith(BASE_PREFIX) && child.isDir()) {
+ long txn = parseBase(p);
+ if (bestBase.status == null) {
+ bestBase.status = child;
+ bestBase.txn = txn;
+ } else if (bestBase.txn < txn) {
+ obsolete.add(bestBase.status);
+ bestBase.status = child;
+ bestBase.txn = txn;
+ } else {
+ obsolete.add(child);
+ }
+ } else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) {
+ ParsedDelta delta = parseDelta(child);
+ if (txnList.isTxnRangeValid(delta.minTransaction,
+ delta.maxTransaction) !=
+ ValidTxnList.RangeResponse.NONE) {
+ working.add(delta);
+ }
+ } else if (child.isDir()) {
+ // This is just the directory. We need to recurse and find the actual files. But don't
+ // do this until we have determined there is no base. This saves time. Plus,
+ // it is possible that the cleaner is running and removing these original files,
+ // in which case recursing through them could cause us to get an error.
+ originalDirectories.add(child);
+ } else {
+ original.add(createOriginalObj(childWithId, child));
+ }
+ }
+
+ public static HdfsFileStatusWithId createOriginalObj(
+ HdfsFileStatusWithId childWithId, FileStatus child) {
+ return childWithId != null ? childWithId : new HdfsFileStatusWithoutId(child);
+ }
+
+ private static class HdfsFileStatusWithoutId implements HdfsFileStatusWithId {
+ private FileStatus fs;
+
+ public HdfsFileStatusWithoutId(FileStatus fs) {
+ this.fs = fs;
+ }
+
+ @Override
+ public FileStatus getFileStatus() {
+ return fs;
+ }
+
+ @Override
+ public Long getFileId() {
+ return null;
+ }
+ }
+
/**
- * Find the original files (non-ACID layout) recursively under the partition
- * directory.
+ * Find the original files (non-ACID layout) recursively under the partition directory.
* @param fs the file system
- * @param stat the file/directory to add
+ * @param stat the directory to add
* @param original the list of original files
* @throws IOException
*/
private static void findOriginals(FileSystem fs, FileStatus stat,
- List<FileStatus> original
- ) throws IOException {
- if (stat.isDir()) {
- for(FileStatus child: SHIMS.listLocatedStatus(fs, stat.getPath(), hiddenFileFilter)) {
- findOriginals(fs, child, original);
+ List<HdfsFileStatusWithId> original, boolean useFileIds) throws IOException {
+ assert stat.isDir();
+ List<HdfsFileStatusWithId> childrenWithId = null;
+ if (useFileIds) {
+ try {
+ childrenWithId = SHIMS.listLocatedHdfsStatus(fs, stat.getPath(), hiddenFileFilter);
+ } catch (Throwable t) {
+ LOG.error("Failed to get files with ID; using regular API", t);
+ useFileIds = false;
+ }
+ }
+ if (childrenWithId != null) {
+ for (HdfsFileStatusWithId child : childrenWithId) {
+ if (child.getFileStatus().isDir()) {
+ findOriginals(fs, child.getFileStatus(), original, useFileIds);
+ } else {
+ original.add(child);
+ }
}
} else {
- original.add(stat);
+ List<FileStatus> children = SHIMS.listLocatedStatus(fs, stat.getPath(), hiddenFileFilter);
+ for (FileStatus child : children) {
+ if (child.isDir()) {
+ findOriginals(fs, child, original, useFileIds);
+ } else {
+ original.add(createOriginalObj(null, child));
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 6ed7872..fd6d2ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -48,12 +48,14 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.io.AcidInputFormat;
+import org.apache.hadoop.hive.ql.io.AcidInputFormat.DeltaMetaData;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.Context;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
@@ -63,6 +65,7 @@ import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
@@ -73,6 +76,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
@@ -436,26 +440,34 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
static final class SplitInfo extends ACIDSplitStrategy {
private final Context context;
private final FileSystem fs;
- private final FileStatus file;
+ private final HdfsFileStatusWithId fileWithId;
private final FileInfo fileInfo;
private final boolean isOriginal;
private final List<DeltaMetaData> deltas;
private final boolean hasBase;
SplitInfo(Context context, FileSystem fs,
- FileStatus file, FileInfo fileInfo,
+ HdfsFileStatusWithId fileWithId, FileInfo fileInfo,
boolean isOriginal,
List<DeltaMetaData> deltas,
boolean hasBase, Path dir, boolean[] covered) throws IOException {
super(dir, context.numBuckets, deltas, covered);
this.context = context;
this.fs = fs;
- this.file = file;
+ this.fileWithId = fileWithId;
this.fileInfo = fileInfo;
this.isOriginal = isOriginal;
this.deltas = deltas;
this.hasBase = hasBase;
}
+
+ @VisibleForTesting
+ public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, FileInfo fileInfo,
+ boolean isOriginal, ArrayList<DeltaMetaData> deltas, boolean hasBase, Path dir,
+ boolean[] covered) throws IOException {
+ this(context, fs, AcidUtils.createOriginalObj(null, fileStatus),
+ fileInfo, isOriginal, deltas, hasBase, dir, covered);
+ }
}
/**
@@ -465,14 +477,15 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
static final class ETLSplitStrategy implements SplitStrategy<SplitInfo> {
Context context;
FileSystem fs;
- List<FileStatus> files;
+ List<HdfsFileStatusWithId> files;
boolean isOriginal;
List<DeltaMetaData> deltas;
Path dir;
boolean[] covered;
- public ETLSplitStrategy(Context context, FileSystem fs, Path dir, List<FileStatus> children,
- boolean isOriginal, List<DeltaMetaData> deltas, boolean[] covered) {
+ public ETLSplitStrategy(Context context, FileSystem fs, Path dir,
+ List<HdfsFileStatusWithId> children, boolean isOriginal, List<DeltaMetaData> deltas,
+ boolean[] covered) {
this.context = context;
this.dir = dir;
this.fs = fs;
@@ -516,14 +529,15 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
@Override
public List<SplitInfo> getSplits() throws IOException {
List<SplitInfo> result = Lists.newArrayList();
- for (FileStatus file : files) {
+ for (HdfsFileStatusWithId file : files) {
FileInfo info = null;
if (context.cacheStripeDetails) {
- info = verifyCachedFileInfo(file);
+ info = verifyCachedFileInfo(file.getFileStatus());
}
// ignore files of 0 length
- if (file.getLen() > 0) {
- result.add(new SplitInfo(context, fs, file, info, isOriginal, deltas, true, dir, covered));
+ if (file.getFileStatus().getLen() > 0) {
+ result.add(new SplitInfo(
+ context, fs, file, info, isOriginal, deltas, true, dir, covered));
}
}
return result;
@@ -540,7 +554,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
* as opposed to query execution (split generation does not read or cache file footers).
*/
static final class BISplitStrategy extends ACIDSplitStrategy {
- List<FileStatus> fileStatuses;
+ List<HdfsFileStatusWithId> fileStatuses;
boolean isOriginal;
List<DeltaMetaData> deltas;
FileSystem fs;
@@ -548,7 +562,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
Path dir;
public BISplitStrategy(Context context, FileSystem fs,
- Path dir, List<FileStatus> fileStatuses, boolean isOriginal,
+ Path dir, List<HdfsFileStatusWithId> fileStatuses, boolean isOriginal,
List<DeltaMetaData> deltas, boolean[] covered) {
super(dir, context.numBuckets, deltas, covered);
this.context = context;
@@ -562,11 +576,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
@Override
public List<OrcSplit> getSplits() throws IOException {
List<OrcSplit> splits = Lists.newArrayList();
- for (FileStatus fileStatus : fileStatuses) {
+ for (HdfsFileStatusWithId file : fileStatuses) {
+ FileStatus fileStatus = file.getFileStatus();
String[] hosts = SHIMS.getLocationsWithOffset(fs, fileStatus).firstEntry().getValue()
.getHosts();
- OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), 0, fileStatus.getLen(), hosts,
- null, isOriginal, true, deltas, -1);
+ OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), file.getFileId(), 0,
+ fileStatus.getLen(), hosts, null, isOriginal, true, deltas, -1);
splits.add(orcSplit);
}
@@ -606,7 +621,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
if (!deltas.isEmpty()) {
for (int b = 0; b < numBuckets; ++b) {
if (!covered[b]) {
- splits.add(new OrcSplit(dir, b, 0, new String[0], null, false, false, deltas, -1));
+ splits.add(new OrcSplit(dir, null, b, 0, new String[0], null, false, false, deltas, -1));
}
}
}
@@ -627,21 +642,23 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private final Context context;
private final FileSystem fs;
private final Path dir;
+ private final boolean useFileIds;
- FileGenerator(Context context, FileSystem fs, Path dir) {
+ FileGenerator(Context context, FileSystem fs, Path dir, boolean useFileIds) {
this.context = context;
this.fs = fs;
this.dir = dir;
+ this.useFileIds = useFileIds;
}
@Override
public SplitStrategy call() throws IOException {
final SplitStrategy splitStrategy;
AcidUtils.Directory dirInfo = AcidUtils.getAcidState(dir,
- context.conf, context.transactionList);
+ context.conf, context.transactionList, useFileIds);
List<DeltaMetaData> deltas = AcidUtils.serializeDeltas(dirInfo.getCurrentDirectories());
Path base = dirInfo.getBaseDirectory();
- List<FileStatus> original = dirInfo.getOriginalFiles();
+ List<HdfsFileStatusWithId> original = dirInfo.getOriginalFiles();
boolean[] covered = new boolean[context.numBuckets];
boolean isOriginal = base == null;
@@ -649,17 +666,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
if (base != null || !original.isEmpty()) {
// find the base files (original or new style)
- List<FileStatus> children = original;
+ List<HdfsFileStatusWithId> children = original;
if (base != null) {
- children = SHIMS.listLocatedStatus(fs, base,
- AcidUtils.hiddenFileFilter);
+ children = findBaseFiles(base, useFileIds);
}
long totalFileSize = 0;
- for (FileStatus child : children) {
- totalFileSize += child.getLen();
+ for (HdfsFileStatusWithId child : children) {
+ totalFileSize += child.getFileStatus().getLen();
AcidOutputFormat.Options opts = AcidUtils.parseBaseBucketFilename
- (child.getPath(), context.conf);
+ (child.getFileStatus().getPath(), context.conf);
int b = opts.getBucket();
// If the bucket is in the valid range, mark it as covered.
// I wish Hive actually enforced bucketing all of the time.
@@ -700,6 +716,24 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
return splitStrategy;
}
+
+ private List<HdfsFileStatusWithId> findBaseFiles(
+ Path base, boolean useFileIds) throws IOException {
+ if (useFileIds) {
+ try {
+ return SHIMS.listLocatedHdfsStatus(fs, base, AcidUtils.hiddenFileFilter);
+ } catch (Throwable t) {
+ LOG.error("Failed to get files with ID; using regular API", t);
+ }
+ }
+ // Fall back to regular API and create states without ID.
+ List<FileStatus> children = SHIMS.listLocatedStatus(fs, base, AcidUtils.hiddenFileFilter);
+ List<HdfsFileStatusWithId> result = new ArrayList<>(children.size());
+ for (FileStatus child : children) {
+ result.add(AcidUtils.createOriginalObj(null, child));
+ }
+ return result;
+ }
}
/**
@@ -709,6 +743,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
static final class SplitGenerator implements Callable<List<OrcSplit>> {
private final Context context;
private final FileSystem fs;
+ private final HdfsFileStatusWithId fileWithId;
private final FileStatus file;
private final long blockSize;
private final TreeMap<Long, BlockLocation> locations;
@@ -728,8 +763,9 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
public SplitGenerator(SplitInfo splitInfo) throws IOException {
this.context = splitInfo.context;
this.fs = splitInfo.fs;
- this.file = splitInfo.file;
- this.blockSize = file.getBlockSize();
+ this.fileWithId = splitInfo.fileWithId;
+ this.file = this.fileWithId.getFileStatus();
+ this.blockSize = this.file.getBlockSize();
this.fileInfo = splitInfo.fileInfo;
locations = SHIMS.getLocationsWithOffset(fs, file);
this.isOriginal = splitInfo.isOriginal;
@@ -837,8 +873,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
final double splitRatio = (double) length / (double) fileLen;
final long scaledProjSize = projColsUncompressedSize > 0 ?
(long) (splitRatio * projColsUncompressedSize) : fileLen;
- return new OrcSplit(file.getPath(), offset, length, hosts, fileMetaInfo,
- isOriginal, hasBase, deltas, scaledProjSize);
+ return new OrcSplit(file.getPath(), fileWithId.getFileId(), offset, length, hosts,
+ fileMetaInfo, isOriginal, hasBase, deltas, scaledProjSize);
}
/**
@@ -1020,9 +1056,10 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
List<Future<?>> splitFutures = Lists.newArrayList();
// multi-threaded file statuses and split strategy
+ boolean useFileIds = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS);
for (Path dir : getInputPaths(conf)) {
FileSystem fs = dir.getFileSystem(conf);
- FileGenerator fileGenerator = new FileGenerator(context, fs, dir);
+ FileGenerator fileGenerator = new FileGenerator(context, fs, dir, useFileIds);
pathFutures.add(context.threadPool.submit(fileGenerator));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index 8cf4cc0..cc03df7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@ -25,6 +25,8 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.AcidInputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -39,6 +41,8 @@ import org.apache.hadoop.mapred.FileSplit;
*
*/
public class OrcSplit extends FileSplit {
+ private static final Log LOG = LogFactory.getLog(OrcSplit.class);
+
private ReaderImpl.FileMetaInfo fileMetaInfo;
private boolean hasFooter;
private boolean isOriginal;
@@ -46,7 +50,9 @@ public class OrcSplit extends FileSplit {
private final List<AcidInputFormat.DeltaMetaData> deltas = new ArrayList<>();
private OrcFile.WriterVersion writerVersion;
private long projColsUncompressedSize;
+ private transient Long fileId;
+ static final int HAS_FILEID_FLAG = 8;
static final int BASE_FLAG = 4;
static final int ORIGINAL_FLAG = 2;
static final int FOOTER_FLAG = 1;
@@ -58,10 +64,13 @@ public class OrcSplit extends FileSplit {
super(null, 0, 0, (String[]) null);
}
- public OrcSplit(Path path, long offset, long length, String[] hosts,
+ public OrcSplit(Path path, Long fileId, long offset, long length, String[] hosts,
ReaderImpl.FileMetaInfo fileMetaInfo, boolean isOriginal, boolean hasBase,
List<AcidInputFormat.DeltaMetaData> deltas, long projectedDataSize) {
super(path, offset, length, hosts);
+ // We could avoid serializing file ID and just replace the path with inode-based path.
+ // However, that breaks bunch of stuff because Hive later looks up things by split path.
+ this.fileId = fileId;
this.fileMetaInfo = fileMetaInfo;
hasFooter = this.fileMetaInfo != null;
this.isOriginal = isOriginal;
@@ -77,7 +86,8 @@ public class OrcSplit extends FileSplit {
int flags = (hasBase ? BASE_FLAG : 0) |
(isOriginal ? ORIGINAL_FLAG : 0) |
- (hasFooter ? FOOTER_FLAG : 0);
+ (hasFooter ? FOOTER_FLAG : 0) |
+ (fileId != null ? HAS_FILEID_FLAG : 0);
out.writeByte(flags);
out.writeInt(deltas.size());
for(AcidInputFormat.DeltaMetaData delta: deltas) {
@@ -99,6 +109,9 @@ public class OrcSplit extends FileSplit {
footerBuff.limit() - footerBuff.position());
WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId());
}
+ if (fileId != null) {
+ out.writeLong(fileId.longValue());
+ }
}
@Override
@@ -110,6 +123,7 @@ public class OrcSplit extends FileSplit {
hasFooter = (FOOTER_FLAG & flags) != 0;
isOriginal = (ORIGINAL_FLAG & flags) != 0;
hasBase = (BASE_FLAG & flags) != 0;
+ boolean hasFileId = (HAS_FILEID_FLAG & flags) != 0;
deltas.clear();
int numDeltas = in.readInt();
@@ -134,6 +148,9 @@ public class OrcSplit extends FileSplit {
fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize,
metadataSize, footerBuff, writerVersion);
}
+ if (hasFileId) {
+ fileId = in.readLong();
+ }
}
ReaderImpl.FileMetaInfo getFileMetaInfo(){
@@ -159,4 +176,8 @@ public class OrcSplit extends FileSplit {
public long getProjectedColumnsUncompressedSize() {
return projColsUncompressedSize;
}
+
+ public Long getFileId() {
+ return fileId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 8e431b2..02fa725 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
@@ -133,7 +134,8 @@ public class CompactorMR {
// and discovering that in getSplits is too late as we then have no way to pass it to our
// mapper.
- AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(sd.getLocation()), conf, txns);
+ AcidUtils.Directory dir = AcidUtils.getAcidState(
+ new Path(sd.getLocation()), conf, txns, false);
StringableList dirsToSearch = new StringableList();
Path baseDir = null;
if (isMajor) {
@@ -141,12 +143,13 @@ public class CompactorMR {
// partition is just now being converted to ACID.
baseDir = dir.getBaseDirectory();
if (baseDir == null) {
- List<FileStatus> originalFiles = dir.getOriginalFiles();
+ List<HdfsFileStatusWithId> originalFiles = dir.getOriginalFiles();
if (!(originalFiles == null) && !(originalFiles.size() == 0)) {
// There are original format files
- for (FileStatus stat : originalFiles) {
- dirsToSearch.add(stat.getPath());
- LOG.debug("Adding original file " + stat.getPath().toString() + " to dirs to search");
+ for (HdfsFileStatusWithId stat : originalFiles) {
+ Path path = stat.getFileStatus().getPath();
+ dirsToSearch.add(path);
+ LOG.debug("Adding original file " + path + " to dirs to search");
}
// Set base to the location so that the input format reads the original files.
baseDir = new Path(sd.getLocation());
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 73715c6..9bf725d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler;
import org.apache.hadoop.hive.metastore.txn.TxnHandler;
import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -223,7 +224,7 @@ public class Initiator extends CompactorThread {
boolean noBase = false;
Path location = new Path(sd.getLocation());
FileSystem fs = location.getFileSystem(conf);
- AcidUtils.Directory dir = AcidUtils.getAcidState(location, conf, txns);
+ AcidUtils.Directory dir = AcidUtils.getAcidState(location, conf, txns, false);
Path base = dir.getBaseDirectory();
long baseSize = 0;
FileStatus stat = null;
@@ -236,9 +237,9 @@ public class Initiator extends CompactorThread {
baseSize = sumDirSize(fs, base);
}
- List<FileStatus> originals = dir.getOriginalFiles();
- for (FileStatus origStat : originals) {
- baseSize += origStat.getLen();
+ List<HdfsFileStatusWithId> originals = dir.getOriginalFiles();
+ for (HdfsFileStatusWithId origStat : originals) {
+ baseSize += origStat.getFileStatus().getLen();
}
long deltaSize = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
index f8ded12..b6ba862 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFile;
import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem;
import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath;
+import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
import org.junit.Test;
import java.util.List;
@@ -102,13 +103,14 @@ public class TestAcidUtils {
assertEquals(null, dir.getBaseDirectory());
assertEquals(0, dir.getCurrentDirectories().size());
assertEquals(0, dir.getObsolete().size());
- List<FileStatus> result = dir.getOriginalFiles();
+ List<HdfsFileStatusWithId> result = dir.getOriginalFiles();
assertEquals(5, result.size());
- assertEquals("mock:/tbl/part1/000000_0", result.get(0).getPath().toString());
- assertEquals("mock:/tbl/part1/000001_1", result.get(1).getPath().toString());
- assertEquals("mock:/tbl/part1/000002_0", result.get(2).getPath().toString());
- assertEquals("mock:/tbl/part1/random", result.get(3).getPath().toString());
- assertEquals("mock:/tbl/part1/subdir/000000_0", result.get(4).getPath().toString());
+ assertEquals("mock:/tbl/part1/000000_0", result.get(0).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/000001_1", result.get(1).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/000002_0", result.get(2).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/random", result.get(3).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/subdir/000000_0",
+ result.get(4).getFileStatus().getPath().toString());
}
@Test
@@ -136,13 +138,14 @@ public class TestAcidUtils {
obsolete.get(0).getPath().toString());
assertEquals("mock:/tbl/part1/delta_029_029",
obsolete.get(1).getPath().toString());
- List<FileStatus> result = dir.getOriginalFiles();
+ List<HdfsFileStatusWithId> result = dir.getOriginalFiles();
assertEquals(5, result.size());
- assertEquals("mock:/tbl/part1/000000_0", result.get(0).getPath().toString());
- assertEquals("mock:/tbl/part1/000001_1", result.get(1).getPath().toString());
- assertEquals("mock:/tbl/part1/000002_0", result.get(2).getPath().toString());
- assertEquals("mock:/tbl/part1/random", result.get(3).getPath().toString());
- assertEquals("mock:/tbl/part1/subdir/000000_0", result.get(4).getPath().toString());
+ assertEquals("mock:/tbl/part1/000000_0", result.get(0).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/000001_1", result.get(1).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/000002_0", result.get(2).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/random", result.get(3).getFileStatus().getPath().toString());
+ assertEquals("mock:/tbl/part1/subdir/000000_0",
+ result.get(4).getFileStatus().getPath().toString());
List<AcidUtils.ParsedDelta> deltas = dir.getCurrentDirectories();
assertEquals(2, deltas.size());
AcidUtils.ParsedDelta delt = deltas.get(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 0c12c89..547e799 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -483,7 +483,7 @@ public class TestInputOutputFormat {
final OrcInputFormat.Context context = new OrcInputFormat.Context(
conf, n);
OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator(
- context, fs, new MockPath(fs, "mock:/a/b"));
+ context, fs, new MockPath(fs, "mock:/a/b"), false);
final SplitStrategy splitStrategy = gen.call();
assertTrue(
String.format(
@@ -507,7 +507,7 @@ public class TestInputOutputFormat {
new MockFile("mock:/a/b/part-04", 1000, new byte[0]));
OrcInputFormat.FileGenerator gen =
new OrcInputFormat.FileGenerator(context, fs,
- new MockPath(fs, "mock:/a/b"));
+ new MockPath(fs, "mock:/a/b"), false);
SplitStrategy splitStrategy = gen.call();
assertEquals(true, splitStrategy instanceof OrcInputFormat.BISplitStrategy);
@@ -520,7 +520,7 @@ public class TestInputOutputFormat {
new MockFile("mock:/a/b/.part-03", 1000, new byte[1000]),
new MockFile("mock:/a/b/part-04", 1000, new byte[1000]));
gen = new OrcInputFormat.FileGenerator(context, fs,
- new MockPath(fs, "mock:/a/b"));
+ new MockPath(fs, "mock:/a/b"), false);
splitStrategy = gen.call();
assertEquals(true, splitStrategy instanceof OrcInputFormat.ETLSplitStrategy);
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index ffffcb7..a56309f 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -722,4 +722,15 @@ public class Hadoop20SShims extends HadoopShimsSecure {
Token<?> fsToken = fs.getDelegationToken(uname);
cred.addToken(fsToken.getService(), fsToken);
}
+
+ @Override
+ public List<HdfsFileStatusWithId> listLocatedHdfsStatus(
+ FileSystem fs, Path path, PathFilter filter) throws IOException {
+ throw new UnsupportedOperationException("Not supported on old version");
+ }
+
+ @Override
+ public long getFileId(FileSystem fs, String path) throws IOException {
+ throw new UnsupportedOperationException("Not supported on old version");
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 9eae0ac..e5be8d6 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -61,10 +61,13 @@ import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.io.LongWritable;
@@ -662,6 +665,64 @@ public class Hadoop23Shims extends HadoopShimsSecure {
return result;
}
+ private static final class HdfsFileStatusWithIdImpl implements HdfsFileStatusWithId {
+ private final LocatedFileStatus lfs;
+ private final long fileId;
+
+ public HdfsFileStatusWithIdImpl(LocatedFileStatus lfs, long fileId) {
+ this.lfs = lfs;
+ this.fileId = fileId;
+ }
+
+ @Override
+ public FileStatus getFileStatus() {
+ return lfs;
+ }
+
+ @Override
+ public Long getFileId() {
+ return fileId;
+ }
+ }
+
+ @Override
+ public List<HdfsFileStatusWithId> listLocatedHdfsStatus(
+ FileSystem fs, Path p, PathFilter filter) throws IOException {
+ DistributedFileSystem dfs = ensureDfs(fs);
+ DFSClient dfsc = dfs.getClient();
+ final String src = p.toUri().getPath();
+ DirectoryListing current = dfsc.listPaths(src,
+ org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true);
+ if (current == null) { // the directory does not exist
+ throw new FileNotFoundException("File " + p + " does not exist.");
+ }
+ final URI fsUri = fs.getUri();
+ List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>(
+ current.getPartialListing().length);
+ while (current != null) {
+ org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing();
+ for (int i = 0; i < hfss.length; ++i) {
+ HdfsLocatedFileStatus next = (HdfsLocatedFileStatus)(hfss[i]);
+ if (filter != null) {
+ Path filterPath = next.getFullPath(p).makeQualified(fsUri, null);
+ if (!filter.accept(filterPath)) continue;
+ }
+ LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p);
+ result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId()));
+ }
+ current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null;
+ }
+ return result;
+ }
+
+ private DistributedFileSystem ensureDfs(FileSystem fs) {
+ if (!(fs instanceof DistributedFileSystem)) {
+ throw new UnsupportedOperationException("Only supported for DFS; got " + fs.getClass());
+ }
+ DistributedFileSystem dfs = (DistributedFileSystem)fs;
+ return dfs;
+ }
+
@Override
public BlockLocation[] getLocations(FileSystem fs,
FileStatus status) throws IOException {
@@ -1352,4 +1413,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
// Use method addDelegationTokens instead of getDelegationToken to get all the tokens including KMS.
fs.addDelegationTokens(uname, cred);
}
+
+ @Override
+ public long getFileId(FileSystem fs, String path) throws IOException {
+ return ensureDfs(fs).getClient().getFileInfo(path).getFileId();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 74785e5..2b6f322 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -256,6 +256,10 @@ public interface HadoopShims {
List<FileStatus> listLocatedStatus(FileSystem fs, Path path,
PathFilter filter) throws IOException;
+
+ List<HdfsFileStatusWithId> listLocatedHdfsStatus(
+ FileSystem fs, Path path, PathFilter filter) throws IOException;
+
/**
* For file status returned by listLocatedStatus, convert them into a list
* of block locations.
@@ -316,6 +320,11 @@ public interface HadoopShims {
public void debugLog();
}
+ public interface HdfsFileStatusWithId {
+ public FileStatus getFileStatus();
+ public Long getFileId();
+ }
+
public HCatHadoopShims getHCatShim();
public interface HCatHadoopShims {
@@ -731,4 +740,10 @@ public interface HadoopShims {
* @throws IOException If an error occurred on adding the token.
*/
public void addDelegationTokens(FileSystem fs, Credentials cred, String uname) throws IOException;
+
+ /**
+ * Gets file ID. Only supported on hadoop-2.
+ * @return inode ID of the file.
+ */
+ long getFileId(FileSystem fs, String path) throws IOException;
}
[28/50] [abbrv] hive git commit: HIVE-11597 : [CBO new return path]
Handling of strings of zero-length (Ashutosh Chauhan via Jesus Camacho
Rodriguez)
Posted by se...@apache.org.
HIVE-11597 : [CBO new return path] Handling of strings of zero-length (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c3c2bdac
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c3c2bdac
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c3c2bdac
Branch: refs/heads/llap
Commit: c3c2bdacdbfbcad5a92959e05313c6ff677af89c
Parents: e1b88a1
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Wed Aug 19 09:15:18 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Aug 19 09:15:18 2015 -0700
----------------------------------------------------------------------
.../ql/optimizer/calcite/translator/TypeConverter.java | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c2bdac/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
index 13c8c23..2825f77 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
@@ -31,6 +31,8 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -300,12 +302,17 @@ public class TypeConverter {
case DECIMAL:
return TypeInfoFactory.getDecimalTypeInfo(rType.getPrecision(), rType.getScale());
case VARCHAR:
- if (rType.getPrecision() == Integer.MAX_VALUE)
+ int varcharLength = rType.getPrecision();
+ if (varcharLength < 1 || varcharLength > HiveVarchar.MAX_VARCHAR_LENGTH)
return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
else
- return TypeInfoFactory.getVarcharTypeInfo(rType.getPrecision());
+ return TypeInfoFactory.getVarcharTypeInfo(varcharLength);
case CHAR:
- return TypeInfoFactory.getCharTypeInfo(rType.getPrecision());
+ int charLength = rType.getPrecision();
+ if (charLength < 1 || charLength > HiveChar.MAX_CHAR_LENGTH)
+ return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
+ else
+ return TypeInfoFactory.getCharTypeInfo(charLength);
case OTHER:
default:
return TypeInfoFactory.voidTypeInfo;
[13/50] [abbrv] hive git commit: HIVE-11567: Some trace logs seeped
through with new log4j2 changes (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-11567: Some trace logs seeped through with new log4j2 changes (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf0481fc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf0481fc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf0481fc
Branch: refs/heads/llap
Commit: cf0481fcf26087dc2cd2de8b10bc2b13befa96ac
Parents: 0fab86c
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 14:04:10 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 14:04:10 2015 -0700
----------------------------------------------------------------------
beeline/src/main/resources/beeline-log4j2.xml | 2 +-
common/src/test/resources/hive-exec-log4j2-test.xml | 2 +-
common/src/test/resources/hive-log4j2-test.xml | 2 +-
data/conf/hive-log4j2.xml | 2 +-
hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml | 2 +-
ql/src/main/resources/hive-exec-log4j2.xml | 2 +-
testutils/ptest2/src/main/resources/log4j2.xml | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/beeline/src/main/resources/beeline-log4j2.xml
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/beeline-log4j2.xml b/beeline/src/main/resources/beeline-log4j2.xml
index 5f09741..2349c5a 100644
--- a/beeline/src/main/resources/beeline-log4j2.xml
+++ b/beeline/src/main/resources/beeline-log4j2.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.threshold">DEBUG</Property>
<Property name="hive.log.level">WARN</Property>
<Property name="hive.root.logger">console</Property>
</Properties>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/common/src/test/resources/hive-exec-log4j2-test.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-exec-log4j2-test.xml b/common/src/test/resources/hive-exec-log4j2-test.xml
index b5f2cb4..1d91b26 100644
--- a/common/src/test/resources/hive-exec-log4j2-test.xml
+++ b/common/src/test/resources/hive-exec-log4j2-test.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.threshold">DEBUG</Property>
<Property name="hive.log.level">INFO</Property>
<Property name="hive.root.logger">FA</Property>
<Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/common/src/test/resources/hive-log4j2-test.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-log4j2-test.xml b/common/src/test/resources/hive-log4j2-test.xml
index 63b46c8..98ca6f8 100644
--- a/common/src/test/resources/hive-log4j2-test.xml
+++ b/common/src/test/resources/hive-log4j2-test.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.threshold">DEBUG</Property>
<Property name="hive.log.level">WARN</Property>
<Property name="hive.root.logger">DRFA</Property>
<Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index ff8e4d3..51173a0 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.threshold">DEBUG</Property>
<Property name="hive.log.level">DEBUG</Property>
<Property name="hive.root.logger">DRFA</Property>
<Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
index 40da974..96f0974 100644
--- a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
+++ b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="webhcat.log.threshold">ALL</Property>
+ <Property name="webhcat.log.threshold">DEBUG</Property>
<Property name="webhcat.log.level">INFO</Property>
<Property name="webhcat.root.logger">standard</Property>
<Property name="webhcat.log.dir">.</Property>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
index c93437c..8b520a2 100644
--- a/ql/src/main/resources/hive-exec-log4j2.xml
+++ b/ql/src/main/resources/hive-exec-log4j2.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.threshold">DEBUG</Property>
<Property name="hive.log.level">INFO</Property>
<Property name="hive.root.logger">FA</Property>
<Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/testutils/ptest2/src/main/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
index 992462e..6502ad1 100644
--- a/testutils/ptest2/src/main/resources/log4j2.xml
+++ b/testutils/ptest2/src/main/resources/log4j2.xml
@@ -20,7 +20,7 @@
packages="org.apache.hadoop.hive.ql.log">
<Properties>
- <Property name="hive.ptest.log.threshold">ALL</Property>
+ <Property name="hive.ptest.log.threshold">DEBUG</Property>
<Property name="hive.ptest.log.level">DEBUG</Property>
<Property name="hive.ptest.root.logger">FILE</Property>
<Property name="hive.ptest.log.dir">target</Property>
[49/50] [abbrv] hive git commit: HIVE-11620: Fix several qtest output
order (Jimmy, reviewed by Chao)
Posted by se...@apache.org.
HIVE-11620: Fix several qtest output order (Jimmy, reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/284859c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/284859c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/284859c4
Branch: refs/heads/llap
Commit: 284859c4edca76e2cd7e7bbecb2b5a0fb753ea06
Parents: 0e54991
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Wed Aug 19 18:04:49 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Tue Aug 25 08:59:07 2015 -0700
----------------------------------------------------------------------
.../queries/clientpositive/selectDistinctStar.q | 2 +
.../clientpositive/unionall_unbalancedppd.q | 3 ++
.../clientpositive/vector_cast_constant.q | 4 +-
.../clientpositive/selectDistinctStar.q.out | 8 +++-
.../spark/vector_cast_constant.q.java1.7.out | 41 +++++++++++++-------
.../clientpositive/tez/selectDistinctStar.q.out | 8 +++-
.../tez/vector_cast_constant.q.java1.7.out | 21 ++++++++--
.../clientpositive/unionall_unbalancedppd.q.out | 14 ++++---
.../vector_cast_constant.q.java1.7.out | 32 ++++++++++++---
9 files changed, 99 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/queries/clientpositive/selectDistinctStar.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/selectDistinctStar.q b/ql/src/test/queries/clientpositive/selectDistinctStar.q
index 244824f..cee83bb 100644
--- a/ql/src/test/queries/clientpositive/selectDistinctStar.q
+++ b/ql/src/test/queries/clientpositive/selectDistinctStar.q
@@ -3,6 +3,8 @@ set hive.explain.user=false;
explain select distinct src.* from src;
+-- SORT_QUERY_RESULTS
+
select distinct src.* from src;
select distinct * from src;
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/queries/clientpositive/unionall_unbalancedppd.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/unionall_unbalancedppd.q b/ql/src/test/queries/clientpositive/unionall_unbalancedppd.q
index ab0a70b..360ad11 100644
--- a/ql/src/test/queries/clientpositive/unionall_unbalancedppd.q
+++ b/ql/src/test/queries/clientpositive/unionall_unbalancedppd.q
@@ -1,4 +1,7 @@
set hive.optimize.ppd=true;
+
+-- SORT_QUERY_RESULTS
+
drop table if exists union_all_bug_test_1;
drop table if exists union_all_bug_test_2;
create table if not exists union_all_bug_test_1
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/queries/clientpositive/vector_cast_constant.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_cast_constant.q b/ql/src/test/queries/clientpositive/vector_cast_constant.q
index d15e87b..5b762c7 100644
--- a/ql/src/test/queries/clientpositive/vector_cast_constant.q
+++ b/ql/src/test/queries/clientpositive/vector_cast_constant.q
@@ -43,11 +43,11 @@ EXPLAIN SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10;
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10;
SELECT
i,
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10;
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10;
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/selectDistinctStar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/selectDistinctStar.q.out b/ql/src/test/results/clientpositive/selectDistinctStar.q.out
index 45d9882..abd6795 100644
--- a/ql/src/test/results/clientpositive/selectDistinctStar.q.out
+++ b/ql/src/test/results/clientpositive/selectDistinctStar.q.out
@@ -51,11 +51,15 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select distinct src.* from src
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+select distinct src.* from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select distinct src.* from src
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+select distinct src.* from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
index 9e1402b..726e8c3 100644
--- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.java1.7.out
@@ -106,14 +106,14 @@ PREHOOK: query: EXPLAIN SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
i,
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -124,6 +124,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -156,6 +157,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4))
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: decimal(14,4))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
@@ -166,6 +178,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -178,7 +191,7 @@ PREHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
PREHOOK: Input: default@over1korc
#### A masked pattern was here ####
@@ -187,17 +200,17 @@ POSTHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
-65636 50.0 50.0 50
-65550 50.0 50.0 50
-65592 50.0 50.0 50
-65744 50.0 50.0 50
-65668 50.0 50.0 50
-65722 50.0 50.0 50
-65598 50.0 50.0 50
-65568 50.0 50.0 50
-65596 50.0 50.0 50
-65738 50.0 50.0 50
+65536 50.0 50.0 50
+65537 50.0 50.0 50
+65538 50.0 50.0 50
+65539 50.0 50.0 50
+65540 50.0 50.0 50
+65541 50.0 50.0 50
+65542 50.0 50.0 50
+65543 50.0 50.0 50
+65544 50.0 50.0 50
+65545 50.0 50.0 50
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out b/ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out
index 590db13..0a091da 100644
--- a/ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out
+++ b/ql/src/test/results/clientpositive/tez/selectDistinctStar.q.out
@@ -57,11 +57,15 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select distinct src.* from src
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+select distinct src.* from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select distinct src.* from src
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+select distinct src.* from src
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/tez/vector_cast_constant.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_cast_constant.q.java1.7.out b/ql/src/test/results/clientpositive/tez/vector_cast_constant.q.java1.7.out
index c700038..ac8e02b 100644
--- a/ql/src/test/results/clientpositive/tez/vector_cast_constant.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/tez/vector_cast_constant.q.java1.7.out
@@ -106,14 +106,14 @@ PREHOOK: query: EXPLAIN SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
i,
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -124,6 +124,7 @@ STAGE PLANS:
Tez
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -156,6 +157,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4))
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: decimal(14,4))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
@@ -166,6 +178,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -178,7 +191,7 @@ PREHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
PREHOOK: Input: default@over1korc
#### A masked pattern was here ####
@@ -187,7 +200,7 @@ POSTHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out b/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
index 88a6f30..1562087 100644
--- a/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
+++ b/ql/src/test/results/clientpositive/unionall_unbalancedppd.q.out
@@ -1,6 +1,10 @@
-PREHOOK: query: drop table if exists union_all_bug_test_1
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists union_all_bug_test_1
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists union_all_bug_test_1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists union_all_bug_test_1
POSTHOOK: type: DROPTABLE
PREHOOK: query: drop table if exists union_all_bug_test_2
PREHOOK: type: DROPTABLE
@@ -283,8 +287,8 @@ POSTHOOK: Input: default@union_all_bug_test_1
POSTHOOK: Input: default@union_all_bug_test_2
#### A masked pattern was here ####
0
-1
0
+1
PREHOOK: query: SELECT f1
FROM (
@@ -325,10 +329,10 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@union_all_bug_test_1
POSTHOOK: Input: default@union_all_bug_test_2
#### A masked pattern was here ####
-1
0
-1
0
+1
+1
PREHOOK: query: explain
SELECT f1
http://git-wip-us.apache.org/repos/asf/hive/blob/284859c4/ql/src/test/results/clientpositive/vector_cast_constant.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_cast_constant.q.java1.7.out b/ql/src/test/results/clientpositive/vector_cast_constant.q.java1.7.out
index 789e6c2..9edd6f1 100644
--- a/ql/src/test/results/clientpositive/vector_cast_constant.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/vector_cast_constant.q.java1.7.out
@@ -106,18 +106,19 @@ PREHOOK: query: EXPLAIN SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
i,
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -150,6 +151,27 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: decimal(14,4))
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: decimal(14,4))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE
@@ -172,7 +194,7 @@ PREHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
PREHOOK: type: QUERY
PREHOOK: Input: default@over1korc
#### A masked pattern was here ####
@@ -181,7 +203,7 @@ POSTHOOK: query: SELECT
AVG(CAST(50 AS INT)) AS `avg_int_ok`,
AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`,
AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok`
- FROM over1korc GROUP BY i LIMIT 10
+ FROM over1korc GROUP BY i ORDER BY i LIMIT 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
[23/50] [abbrv] hive git commit: HIVE-11575: Fix test failures in
master due to log4j changes (Prasanth Jayachandran reviewed by Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-11575: Fix test failures in master due to log4j changes (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e0594099
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e0594099
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e0594099
Branch: refs/heads/llap
Commit: e0594099c2ea1652e49ff041c6dfb569f47a5912
Parents: 9d8515d
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Aug 17 13:47:09 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Aug 17 13:47:09 2015 -0700
----------------------------------------------------------------------
data/conf/hive-log4j2.xml | 5 ++---
data/conf/hive-site.xml | 6 ------
.../service/cli/operation/TestOperationLoggingAPIWithMr.java | 2 --
.../service/cli/operation/TestOperationLoggingAPIWithTez.java | 2 --
.../hive/service/cli/operation/TestOperationLoggingLayout.java | 2 --
5 files changed, 2 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index 51173a0..11c8e79 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -25,7 +25,6 @@
<Property name="hive.root.logger">DRFA</Property>
<Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>
<Property name="hive.log.file">hive.log</Property>
- <Property name="hive.ql.log.PerfLogger.level">INFO</Property>
</Properties>
<Appenders>
@@ -95,8 +94,8 @@
<Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
<AppenderRef ref="${sys:hive.root.logger}"/>
</Logger>
- <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}" additivity="false">
- <AppenderRef ref="${sys:hive.root.logger}"/>
+ <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
+ <AppenderRef ref="${sys:hive.ql.log.PerfLogger.logger}"/>
</Logger>
<Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
<AppenderRef ref="${sys:hive.root.logger}"/>
http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/data/conf/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
index 8f7fb28..a58017e 100644
--- a/data/conf/hive-site.xml
+++ b/data/conf/hive-site.xml
@@ -241,12 +241,6 @@
</property>
<property>
- <name>hive.ql.log.PerfLogger.level</name>
- <value>WARN,DRFA</value>
- <description>Used to change the perflogger level</description>
-</property>
-
-<property>
<name>hive.fetch.task.conversion</name>
<value>minimal</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
index 920b563..0155b75 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
@@ -62,8 +62,6 @@ public class TestOperationLoggingAPIWithMr extends OperationLoggingAPITestBase{
};
hiveConf = new HiveConf();
hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
- // We need to set the below parameter to test performance level logging
- hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
miniHS2 = new MiniHS2(hiveConf);
confOverlay = new HashMap<String, String>();
confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
index 31f34b2..ab29861 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
@@ -40,8 +40,6 @@ public class TestOperationLoggingAPIWithTez extends OperationLoggingAPITestBase
};
hiveConf = new HiveConf();
hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
- // We need to set the below parameter to test performance level logging
- hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
// Change the engine to tez
hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
// Set tez execution summary to false.
http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
index 93c16de..56f6a31 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
@@ -39,8 +39,6 @@ public class TestOperationLoggingLayout {
tableName = "TestOperationLoggingLayout_table";
hiveConf = new HiveConf();
hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "execution");
- // We need to set the below parameter to test performance level logging
- hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
miniHS2 = new MiniHS2(hiveConf);
confOverlay = new HashMap<String, String>();
confOverlay.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
[20/50] [abbrv] hive git commit: HIVE-11557: CBO (Calcite Return
Path): Convert to flat AND/OR (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11557: CBO (Calcite Return Path): Convert to flat AND/OR (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8329ee0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8329ee0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8329ee0
Branch: refs/heads/llap
Commit: e8329ee0068dbfe2ca92399d0109f308d6cf7541
Parents: 147347a
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Sat Aug 15 11:11:42 2015 +0300
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Sat Aug 15 11:11:42 2015 +0300
----------------------------------------------------------------------
.../calcite/translator/ExprNodeConverter.java | 26 --------------------
1 file changed, 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e8329ee0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
index 00bf009..ec22f1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
-import java.util.ArrayList;
import java.util.Calendar;
import java.util.LinkedList;
import java.util.List;
@@ -43,11 +42,9 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema;
@@ -139,29 +136,6 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
&& SqlTypeUtil.equalSansNullability(dTFactory, call.getType(),
call.operands.get(0).getType())) {
return args.get(0);
- } else if (ASTConverter.isFlat(call)) {
- // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the
- // exprnode
- GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), 2);
- ArrayList<ExprNodeDesc> tmpExprArgs = new ArrayList<ExprNodeDesc>();
- tmpExprArgs.addAll(args.subList(0, 2));
- try {
- gfDesc = ExprNodeGenericFuncDesc.newInstance(hiveUdf, tmpExprArgs);
- } catch (UDFArgumentException e) {
- LOG.error(e);
- throw new RuntimeException(e);
- }
- for (int i = 2; i < call.operands.size(); i++) {
- tmpExprArgs = new ArrayList<ExprNodeDesc>();
- tmpExprArgs.add(gfDesc);
- tmpExprArgs.add(args.get(i));
- try {
- gfDesc = ExprNodeGenericFuncDesc.newInstance(hiveUdf, tmpExprArgs);
- } catch (UDFArgumentException e) {
- LOG.error(e);
- throw new RuntimeException(e);
- }
- }
} else {
GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(),
args.size());
[36/50] [abbrv] hive git commit: HIVE-11375: Broken processing of
queries containing NOT (x IS NOT NULL and x <> 0) (Aihua Xu,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11375: Broken processing of queries containing NOT (x IS NOT NULL and x <> 0) (Aihua Xu, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1626749
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1626749
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1626749
Branch: refs/heads/llap
Commit: a16267491978a35f4dd0478405b88a74725050a5
Parents: e3664e2
Author: Aihua Xu <ai...@gmail.com>
Authored: Fri Aug 21 10:39:21 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 21 10:39:21 2015 -0700
----------------------------------------------------------------------
.../optimizer/ConstantPropagateProcFactory.java | 100 ++++-
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 14 +-
.../hive/ql/udf/generic/GenericUDFOPAnd.java | 4 +
.../hive/ql/udf/generic/GenericUDFOPEqual.java | 4 +
.../generic/GenericUDFOPEqualOrGreaterThan.java | 4 +
.../generic/GenericUDFOPEqualOrLessThan.java | 4 +
.../ql/udf/generic/GenericUDFOPGreaterThan.java | 4 +
.../ql/udf/generic/GenericUDFOPLessThan.java | 4 +
.../ql/udf/generic/GenericUDFOPNotEqual.java | 5 +
.../ql/udf/generic/GenericUDFOPNotNull.java | 4 +
.../hive/ql/udf/generic/GenericUDFOPNull.java | 4 +
.../hive/ql/udf/generic/GenericUDFOPOr.java | 4 +
.../queries/clientpositive/folder_predicate.q | 32 ++
.../clientpositive/annotate_stats_filter.q.out | 10 +-
.../results/clientpositive/decimal_udf.q.out | 18 +-
.../clientpositive/filter_cond_pushdown.q.out | 32 +-
.../clientpositive/filter_join_breaktask.q.out | 12 +-
.../test/results/clientpositive/fold_when.q.out | 16 +-
.../clientpositive/folder_predicate.q.out | 368 +++++++++++++++++++
.../clientpositive/input_testxpath2.q.out | 2 +-
.../list_bucket_query_oneskew_3.q.out | 6 +-
.../clientpositive/rand_partitionpruner3.q.out | 12 +-
.../clientpositive/select_unquote_not.q.out | 8 +-
.../spark/filter_join_breaktask.q.out | 12 +-
.../tez/filter_join_breaktask.q.out | 12 +-
.../clientpositive/tez/vector_decimal_udf.q.out | 24 +-
.../clientpositive/udf_isnull_isnotnull.q.out | 2 +-
.../test/results/clientpositive/udf_size.q.out | 2 +-
.../clientpositive/vector_decimal_udf.q.out | 24 +-
29 files changed, 640 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index 55ad0ce..5c6a6df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -70,6 +71,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
@@ -94,6 +99,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -223,6 +229,65 @@ public final class ConstantPropagateProcFactory {
}
/**
+ * Combines the logical not() operator with the child operator if possible.
+ * @param desc the expression to be evaluated
+ * @return the new expression to be replaced
+ * @throws UDFArgumentException
+ */
+ private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException {
+ if (desc instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
+
+ GenericUDF udf = funcDesc.getGenericUDF();
+ if (udf instanceof GenericUDFOPNot) {
+ ExprNodeDesc child = funcDesc.getChildren().get(0);
+ if (child instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc)child;
+ GenericUDF childUDF = childDesc.getGenericUDF();
+ List<ExprNodeDesc> grandChildren = child.getChildren();
+
+ if (childUDF instanceof GenericUDFBaseCompare ||
+ childUDF instanceof GenericUDFOPNull ||
+ childUDF instanceof GenericUDFOPNotNull) {
+ List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
+ for(ExprNodeDesc grandChild : grandChildren) {
+ newGrandChildren.add(foldNegative(grandChild));
+ }
+
+ return ExprNodeGenericFuncDesc.newInstance(
+ childUDF.negative(),
+ newGrandChildren);
+ } else if (childUDF instanceof GenericUDFOPAnd ||
+ childUDF instanceof GenericUDFOPOr) {
+ List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
+ for(ExprNodeDesc grandChild : grandChildren) {
+ newGrandChildren.add(foldNegative(
+ ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(),
+ Arrays.asList(grandChild))));
+ }
+
+ return ExprNodeGenericFuncDesc.newInstance(
+ childUDF.negative(),
+ newGrandChildren);
+ }else if (childUDF instanceof GenericUDFOPNot) {
+ return foldNegative(child.getChildren().get(0));
+ } else {
+ // For operator like if() that cannot be handled, leave not() as it
+ // is and continue processing the children
+ List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
+ for(ExprNodeDesc grandChild : grandChildren) {
+ newGrandChildren.add(foldNegative(grandChild));
+ }
+ childDesc.setChildren(newGrandChildren);
+ return funcDesc;
+ }
+ }
+ }
+ }
+ return desc;
+ }
+
+ /**
* Fold input expression desc, only performing short-cutting.
*
* Unnecessary AND/OR operations involving a constant true/false value will be eliminated.
@@ -238,6 +303,11 @@ public final class ConstantPropagateProcFactory {
private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
+ // Combine NOT operator with the child operator. Otherwise, the following optimization
+ // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
+ // should not be optimized to not(x > 3), but (x <=3 or x is null).
+ desc = foldNegative(desc);
+
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
@@ -249,10 +319,10 @@ public final class ConstantPropagateProcFactory {
newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
}
- // Don't evalulate nondeterministic function since the value can only calculate during runtime.
+ // Don't evaluate nondeterministic function since the value can only calculate during runtime.
if (!isDeterministicUdf(udf)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulating immediately.");
+ LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulate immediately.");
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
return desc;
@@ -276,11 +346,11 @@ public final class ConstantPropagateProcFactory {
*
* This function recursively checks if any subexpression of a specified expression
* can be evaluated to be constant and replaces such subexpression with the constant.
- * If the expression is a derterministic UDF and all the subexpressions are constants,
+ * If the expression is a deterministic UDF and all the subexpressions are constants,
* the value will be calculated immediately (during compilation time vs. runtime).
* e.g.:
* concat(year, month) => 200112 for year=2001, month=12 since concat is deterministic UDF
- * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nonderministic UDF
+ * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nondeterministic UDF
* @param desc folding expression
* @param constants current propagated constant map
* @param cppCtx
@@ -292,6 +362,11 @@ public final class ConstantPropagateProcFactory {
private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
boolean propagate) throws UDFArgumentException {
+ // Combine NOT operator with the child operator. Otherwise, the following optimization
+ // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
+ // should not be optimized to not(x > 3), but (x <=3 or x is null).
+ desc = foldNegative(desc);
+
if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
@@ -303,10 +378,10 @@ public final class ConstantPropagateProcFactory {
newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
}
- // Don't evalulate nondeterministic function since the value can only calculate during runtime.
+ // Don't evaluate nondeterministic function since the value can only calculate during runtime.
if (!isDeterministicUdf(udf)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulating immediately.");
+ LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately.");
}
((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
return desc;
@@ -460,7 +535,6 @@ public final class ConstantPropagateProcFactory {
private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs,
Operator<? extends Serializable> op) throws UDFArgumentException {
-
if (udf instanceof GenericUDFOPEqual) {
assert newExprs.size() == 2;
boolean foundUDFInFirst = false;
@@ -511,6 +585,7 @@ public final class ConstantPropagateProcFactory {
return null;
}
}
+
if (udf instanceof GenericUDFOPAnd) {
final BitSet positionsToRemove = new BitSet();
final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
@@ -532,9 +607,13 @@ public final class ConstantPropagateProcFactory {
childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
notNullExprs.add(childExpr.getChildren().get(0));
notNullExprsPositions.add(i);
- } else if (childExpr instanceof ExprNodeGenericFuncDesc
- && ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFBaseCompare
- && childExpr.getChildren().size() == 2) {
+ } else if (childExpr instanceof ExprNodeGenericFuncDesc &&
+ ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFBaseCompare &&
+ !(((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) &&
+ childExpr.getChildren().size() == 2) {
+ // Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
+ // where <op> can be "=", ">=", "<=", ">", "<".
+ // Note: (key <> 86) and (key is not null) cannot be folded
ExprNodeColumnDesc colDesc = getColumnExpr(childExpr.getChildren().get(0));
if (null == colDesc) {
colDesc = getColumnExpr(childExpr.getChildren().get(1));
@@ -646,6 +725,7 @@ public final class ConstantPropagateProcFactory {
}
}
}
+
if (udf instanceof GenericUDFCase) {
// HIVE-9644 Attempt to fold expression like :
// where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end);
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
index e8df5d3..259fde8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
@@ -213,12 +213,24 @@ public abstract class GenericUDF implements Closeable {
}
/**
- * Some functions are affected by appearing order of arguments (comparisons, for example)
+ * Some functions like comparisons may be affected by appearing order of arguments.
+ * This is to convert a function, such as 3 > x to x < 3. The flip function of
+ * GenericUDFOPGreaterThan is GenericUDFOPLessThan.
*/
public GenericUDF flip() {
return this;
}
+ /**
+ * Gets the negative function of the current one. E.g., GenericUDFOPNotEqual for
+ * GenericUDFOPEqual, or GenericUDFOPNull for GenericUDFOPNotNull.
+ * @return Negative function
+ * @throws UDFArgumentException
+ */
+ public GenericUDF negative() {
+ throw new UnsupportedOperationException("Negative function doesn't exist for " + getFuncName());
+ }
+
public String getUdfName() {
return getClass().getName();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
index db7fbac..eb33d98 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
@@ -97,4 +97,8 @@ public class GenericUDFOPAnd extends GenericUDF {
return sb.toString();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPOr();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
index 428ced7..23f7f1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
@@ -122,4 +122,8 @@ public class GenericUDFOPEqual extends GenericUDFBaseCompare {
return result;
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPNotEqual();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
index d9556cc..bd5b61a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
@@ -134,4 +134,8 @@ public class GenericUDFOPEqualOrGreaterThan extends GenericUDFBaseCompare {
return new GenericUDFOPEqualOrLessThan();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPLessThan();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
index 1d9eaf6..ad971cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
@@ -134,5 +134,9 @@ public class GenericUDFOPEqualOrLessThan extends GenericUDFBaseCompare {
return new GenericUDFOPEqualOrGreaterThan();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPGreaterThan();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
index 8e1f2b1..85f0601 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
@@ -134,5 +134,9 @@ public class GenericUDFOPGreaterThan extends GenericUDFBaseCompare {
return new GenericUDFOPLessThan();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPEqualOrLessThan();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
index 101b348..b445ae6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
@@ -134,4 +134,8 @@ public class GenericUDFOPLessThan extends GenericUDFBaseCompare {
return new GenericUDFOPGreaterThan();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPEqualOrGreaterThan();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
index b5da57a..ac8ec2f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
@@ -121,4 +121,9 @@ public class GenericUDFOPNotEqual extends GenericUDFBaseCompare {
}
return result;
}
+
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPEqual();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
index d22b35d..2b67c38 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
@@ -60,4 +60,8 @@ public class GenericUDFOPNotNull extends GenericUDF {
return children[0] + " is not null";
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPNull();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
index fe20e9a..4eb92eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
@@ -60,4 +60,8 @@ public class GenericUDFOPNull extends GenericUDF {
return children[0] + " is null";
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPNotNull();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
index 4160610..8de59c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
@@ -98,4 +98,8 @@ public class GenericUDFOPOr extends GenericUDF {
return sb.toString();
}
+ @Override
+ public GenericUDF negative() {
+ return new GenericUDFOPAnd();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/queries/clientpositive/folder_predicate.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/folder_predicate.q b/ql/src/test/queries/clientpositive/folder_predicate.q
new file mode 100644
index 0000000..2377dd4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/folder_predicate.q
@@ -0,0 +1,32 @@
+drop table if exists predicate_fold_tb;
+
+create table predicate_fold_tb(value int);
+insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3);
+
+explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3);
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3);
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
index af1e1c3..9e0e78a 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
@@ -297,15 +297,15 @@ STAGE PLANS:
alias: loc_orc
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (not zip is not null) (type: boolean)
+ predicate: zip is null (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
+ expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -375,7 +375,7 @@ STAGE PLANS:
alias: loc_orc
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (not zip is null) (type: boolean)
+ predicate: zip is not null (type: boolean)
Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out
index f36670c..338b754 100644
--- a/ql/src/test/results/clientpositive/decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/decimal_udf.q.out
@@ -973,12 +973,12 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <> 0) (type: boolean)
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key is not null and (key <> 0)) (type: boolean)
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / key) (type: decimal(38,24))
outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0
@@ -1039,12 +1039,12 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21))
outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0
@@ -1095,12 +1095,12 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
index e09057a..af42d5c 100644
--- a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
@@ -37,7 +37,7 @@ STAGE PLANS:
alias: f
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
+ predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -86,17 +86,17 @@ STAGE PLANS:
alias: f
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> '') (type: boolean)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> '')) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -105,14 +105,14 @@ STAGE PLANS:
0 _col3 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col4
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -163,7 +163,7 @@ STAGE PLANS:
alias: f
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
+ predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
@@ -212,17 +212,17 @@ STAGE PLANS:
alias: f
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> '') (type: boolean)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> '')) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: value (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -231,14 +231,14 @@ STAGE PLANS:
0 _col3 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0, _col4
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
index 948ad0d..83f4720 100644
--- a/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
+++ b/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
@@ -271,13 +271,13 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (value <> '') (type: boolean)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((value <> '') and value is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: value (type: string)
sort order: +
Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
tag: 1
auto parallelism: false
Path -> Alias:
@@ -360,17 +360,17 @@ STAGE PLANS:
0 _col7 (type: string)
1 value (type: string)
outputColumnNames: _col0, _col13
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col13 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/fold_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_when.q.out b/ql/src/test/results/clientpositive/fold_when.q.out
index 21748da..bf61a7d 100644
--- a/ql/src/test/results/clientpositive/fold_when.q.out
+++ b/ql/src/test/results/clientpositive/fold_when.q.out
@@ -328,15 +328,15 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not (key = '238')) (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key <> '238') (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -435,15 +435,15 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not (key = '11')) (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key <> '11') (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/folder_predicate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/folder_predicate.q.out b/ql/src/test/results/clientpositive/folder_predicate.q.out
new file mode 100644
index 0000000..1bc1afb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/folder_predicate.q.out
@@ -0,0 +1,368 @@
+PREHOOK: query: drop table if exists predicate_fold_tb
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists predicate_fold_tb
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table predicate_fold_tb(value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@predicate_fold_tb
+POSTHOOK: query: create table predicate_fold_tb(value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@predicate_fold_tb
+PREHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@predicate_fold_tb
+POSTHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@predicate_fold_tb
+POSTHOOK: Lineage: predicate_fold_tb.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value <> 3)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+1
+2
+4
+5
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value < 3)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+1
+2
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value > 3)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+4
+5
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value <= 3)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+1
+2
+3
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value >= 3)) (type: boolean)
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+3
+4
+5
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value = 3)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+3
+PREHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: predicate_fold_tb
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is null or (value <= 1) or (value > 3)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@predicate_fold_tb
+#### A masked pattern was here ####
+NULL
+1
+4
+5
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/input_testxpath2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_testxpath2.q.out b/ql/src/test/results/clientpositive/input_testxpath2.q.out
index d3a6f29..67e270f 100644
--- a/ql/src/test/results/clientpositive/input_testxpath2.q.out
+++ b/ql/src/test/results/clientpositive/input_testxpath2.q.out
@@ -32,7 +32,7 @@ STAGE PLANS:
alias: src_thrift
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (lint is not null and (not mstringstring is null)) (type: boolean)
+ predicate: (lint is not null and mstringstring is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
index fabd0d6..bef079b 100644
--- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
+++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
@@ -293,12 +293,12 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (not (x = 86)) (type: boolean)
- Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ predicate: (x <> 86) (type: boolean)
+ Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: x (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: -- List Bucketing Query
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
index ef8ee2f..634e171 100644
--- a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
+++ b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
@@ -115,12 +115,12 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: ((rand(1) < 0.1) and (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0)))) (type: boolean)
- Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean)
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2'
@@ -247,12 +247,12 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0))) (type: boolean)
- Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 168 Data size: 1784 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
ListSink
PREHOOK: query: select a.* from srcpart a where a.ds = '2008-04-08' and not(key > 50 or key < 10) and a.hr like '%2'
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/select_unquote_not.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/select_unquote_not.q.out b/ql/src/test/results/clientpositive/select_unquote_not.q.out
index 64de433..3b8146b 100644
--- a/ql/src/test/results/clientpositive/select_unquote_not.q.out
+++ b/ql/src/test/results/clientpositive/select_unquote_not.q.out
@@ -59,15 +59,15 @@ STAGE PLANS:
alias: npe_test
Statistics: Num rows: 498 Data size: 5290 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (not (UDFToDouble(ds) < 1970.0)) (type: boolean)
- Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE
+ predicate: (UDFToDouble(ds) >= 1970.0) (type: boolean)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string), ds (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 332 Data size: 3526 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/spark/filter_join_breaktask.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/spark/filter_join_breaktask.q.out
index 28eeb0d..7c3c581 100644
--- a/ql/src/test/results/clientpositive/spark/filter_join_breaktask.q.out
+++ b/ql/src/test/results/clientpositive/spark/filter_join_breaktask.q.out
@@ -289,13 +289,13 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (value <> '') (type: boolean)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((value <> '') and value is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: value (type: string)
sort order: +
Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
tag: 1
auto parallelism: false
Path -> Alias:
@@ -377,17 +377,17 @@ STAGE PLANS:
0 _col7 (type: string)
1 value (type: string)
outputColumnNames: _col0, _col13
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col13 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
index 06f1ca4..99120bb 100644
--- a/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
+++ b/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
@@ -289,13 +289,13 @@ STAGE PLANS:
GatherStats: false
Filter Operator
isSamplingPred: false
- predicate: (value <> '') (type: boolean)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((value <> '') and value is not null) (type: boolean)
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: value (type: string)
sort order: +
Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
tag: 1
auto parallelism: true
Path -> Alias:
@@ -379,17 +379,17 @@ STAGE PLANS:
1 value (type: string)
outputColumnNames: _col0, _col13
Position of Big Table: 0
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col13 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.TextInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
index b24ebcc..13f0de2 100644
--- a/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
@@ -1239,15 +1239,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key is not null and (key <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / key) (type: decimal(38,24))
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1321,15 +1321,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21))
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1393,15 +1393,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
index d459ce3..b59a5b9 100644
--- a/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
+++ b/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
@@ -97,7 +97,7 @@ STAGE PLANS:
alias: src_thrift
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (lint is not null and (not mstringstring is null)) (type: boolean)
+ predicate: (lint is not null and mstringstring is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: lint is not null (type: boolean), lintstring is not null (type: boolean), mstringstring is not null (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/udf_size.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_size.q.out b/ql/src/test/results/clientpositive/udf_size.q.out
index efb06a1..13594d2 100644
--- a/ql/src/test/results/clientpositive/udf_size.q.out
+++ b/ql/src/test/results/clientpositive/udf_size.q.out
@@ -38,7 +38,7 @@ STAGE PLANS:
alias: src_thrift
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (lint is not null and (not mstringstring is null)) (type: boolean)
+ predicate: (lint is not null and mstringstring is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int), -1 (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
index e381757..91a585c 100644
--- a/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
+++ b/ql/src/test/results/clientpositive/vector_decimal_udf.q.out
@@ -1191,15 +1191,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (key <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key is not null and (key <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / key) (type: decimal(38,24))
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1270,15 +1270,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21))
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1339,15 +1339,15 @@ STAGE PLANS:
alias: decimal_udf
Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (value <> 0) (type: boolean)
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ predicate: (value is not null and (value <> 0)) (type: boolean)
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
[27/50] [abbrv] hive git commit: HIVE-11569 : Use PreOrderOnceWalker
where feasible (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Posted by se...@apache.org.
HIVE-11569 : Use PreOrderOnceWalker where feasible (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e1b88a19
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e1b88a19
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e1b88a19
Branch: refs/heads/llap
Commit: e1b88a19a98534cb7c2f6d46e39f5eb135ab6b50
Parents: 6433c3b
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Tue Aug 18 09:13:59 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Aug 18 09:13:59 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/index/RewriteCanApplyCtx.java | 8 ++++----
.../optimizer/physical/NullScanTaskDispatcher.java | 6 +++---
.../hive/ql/optimizer/physical/Vectorizer.java | 16 +++-------------
.../hadoop/hive/ql/ppd/SyntheticJoinPredicate.java | 4 ++--
4 files changed, 12 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
index b56b608..988bb29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -75,7 +75,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
private String baseTableName;
private String indexTableName;
private String aggFunction;
-
+
private TableScanOperator tableScanOperator;
private List<SelectOperator> selectOperators;
private List<GroupByOperator> groupByOperators;
@@ -156,7 +156,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
/**
* This method walks all the nodes starting from topOp TableScanOperator node
* and invokes methods from {@link RewriteCanApplyProcFactory} for each of the rules
- * added to the opRules map. We use the {@link PreOrderWalker} for a pre-order
+ * added to the opRules map. We use the {@link PreOrderOnceWalker} for a pre-order
* traversal of the operator tree.
*
* The methods from {@link RewriteCanApplyProcFactory} set appropriate values in
@@ -180,7 +180,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of topop nodes
List<Node> topNodes = new ArrayList<Node>();
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
index f901812..8dcd283 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer.WalkerCtx;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -61,7 +61,7 @@ import org.apache.hadoop.hive.serde2.NullStructSerDe;
public class NullScanTaskDispatcher implements Dispatcher {
static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName());
-
+
private final PhysicalContext physicalContext;
private final Map<Rule, NodeProcessor> rules;
@@ -177,7 +177,7 @@ public class NullScanTaskDispatcher implements Dispatcher {
// The dispatcher fires the processor corresponding to the closest
// matching rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of topOp nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 7ecd50a..2e3bd76 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyLongOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyMultiKeyOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyStringOperator;
@@ -68,6 +67,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
@@ -155,7 +155,6 @@ public class Vectorizer implements PhysicalPlanResolver {
Set<String> supportedAggregationUdfs = new HashSet<String>();
- private PhysicalContext physicalContext = null;
private HiveConf hiveConf;
public Vectorizer() {
@@ -307,13 +306,10 @@ public class Vectorizer implements PhysicalPlanResolver {
class VectorizationDispatcher implements Dispatcher {
- private final PhysicalContext physicalContext;
-
private List<String> reduceColumnNames;
private List<TypeInfo> reduceTypeInfos;
public VectorizationDispatcher(PhysicalContext physicalContext) {
- this.physicalContext = physicalContext;
reduceColumnNames = null;
reduceTypeInfos = null;
}
@@ -428,7 +424,7 @@ public class Vectorizer implements PhysicalPlanResolver {
MapWorkVectorizationNodeProcessor vnp = new MapWorkVectorizationNodeProcessor(mapWork, isTez);
addMapWorkRules(opRules, vnp);
Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// iterator the mapper operator tree
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(mapWork.getAliasToWork().values());
@@ -701,12 +697,10 @@ public class Vectorizer implements PhysicalPlanResolver {
class MapWorkVectorizationNodeProcessor extends VectorizationNodeProcessor {
- private final MapWork mWork;
private final boolean isTez;
public MapWorkVectorizationNodeProcessor(MapWork mWork, boolean isTez) {
super();
- this.mWork = mWork;
this.isTez = isTez;
}
@@ -765,7 +759,7 @@ public class Vectorizer implements PhysicalPlanResolver {
private final List<String> reduceColumnNames;
private final List<TypeInfo> reduceTypeInfos;
- private boolean isTez;
+ private final boolean isTez;
private Operator<? extends OperatorDesc> rootVectorOp;
@@ -865,7 +859,6 @@ public class Vectorizer implements PhysicalPlanResolver {
@Override
public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
- this.physicalContext = physicalContext;
hiveConf = physicalContext.getConf();
boolean vectorPath = HiveConf.getBoolVar(hiveConf,
@@ -1406,7 +1399,6 @@ public class Vectorizer implements PhysicalPlanResolver {
int[] smallTableIndices;
int smallTableIndicesSize;
- List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable);
if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) {
smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable);
LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices " + Arrays.toString(smallTableIndices));
@@ -1445,8 +1437,6 @@ public class Vectorizer implements PhysicalPlanResolver {
Operator<? extends OperatorDesc> vectorOp = null;
Class<? extends Operator<?>> opClass = null;
- boolean isOuterJoin = !desc.getNoOuterJoin();
-
VectorMapJoinDesc.HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE;
VectorMapJoinDesc.HashTableKind hashTableKind = HashTableKind.NONE;
VectorMapJoinDesc.HashTableKeyType hashTableKeyType = HashTableKeyType.NONE;
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
index 32af813..44159c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.optimizer.Transform;
@@ -92,7 +92,7 @@ public class SyntheticJoinPredicate implements Transform {
// rule and passes the context along
SyntheticContext context = new SyntheticContext(pctx);
Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of top op nodes
List<Node> topNodes = new ArrayList<Node>();
[47/50] [abbrv] hive git commit: HIVE-11469 : Update doc for
InstanceCache to clearly define the contract on the SeedObject (Swarnim
Kulkarni via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11469 : Update doc for InstanceCache to clearly define the contract on the SeedObject (Swarnim Kulkarni via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dd2bdfc6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dd2bdfc6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dd2bdfc6
Branch: refs/heads/llap
Commit: dd2bdfc6702f8ecb88255c4258e4449aa9b75af5
Parents: 7796dd6
Author: Swarnim Kulkarni <ku...@gmail.com>
Authored: Fri Aug 7 09:53:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 24 10:30:16 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/serde2/avro/InstanceCache.java | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dd2bdfc6/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
index c7289cb..8f6a911 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java
@@ -17,20 +17,21 @@
*/
package org.apache.hadoop.hive.serde2.avro;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
- * Cache for objects whose creation only depends on some other set of objects
- * and therefore can be used against other equivalent versions of those
- * objects. Essentially memoizes instance creation.
+ * Cache for objects whose creation only depends on some other set of objects and therefore can be
+ * used against other equivalent versions of those objects. Essentially memoizes instance creation.
*
- * @param <SeedObject> Object that determines the instance
- * @param <Instance> Instance that will be created from SeedObject.
+ * @param <SeedObject> Object that determines the instance. The cache uses this object as a key for
+ * its hash which is why it is imperative to have appropriate equals and hashcode
+ * implementation for this object for the cache to work properly
+ * @param <Instance> Instance that will be created from SeedObject.
*/
public abstract class InstanceCache<SeedObject, Instance> {
private static final Log LOG = LogFactory.getLog(InstanceCache.class);
[44/50] [abbrv] hive git commit: HIVE-10631 : create_table_core
method has invalid update for Fast Stats (Aaron Tokhy via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-10631 : create_table_core method has invalid update for Fast Stats (Aaron Tokhy via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d3d3d01
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d3d3d01
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d3d3d01
Branch: refs/heads/llap
Commit: 9d3d3d0134fed704684fdefa9faa6ed949615cc0
Parents: a16bbd4
Author: Aaron Tokhy <at...@gmail.com>
Authored: Thu Aug 13 10:45:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 24 10:12:18 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveAlterHandler.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 6 +-----
.../hadoop/hive/metastore/MetaStoreUtils.java | 17 ++++++++++-------
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
4 files changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 99d6680..ee20430 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -236,7 +236,7 @@ public class HiveAlterHandler implements AlterHandler {
Database db = msdb.getDatabase(newt.getDbName());
// Update table stats. For partitioned table, we update stats in
// alterPartition()
- MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true);
+ MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true);
}
updateTableColumnStatsForAlterTable(msdb, oldt, newt);
// now finally call alter table
http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index ae500bf..1840e76 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1408,11 +1408,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) &&
!MetaStoreUtils.isView(tbl)) {
- if (tbl.getPartitionKeysSize() == 0) { // Unpartitioned table
- MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir);
- } else { // Partitioned table with no partitions.
- MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, true);
- }
+ MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir);
}
// set create time
http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 907cbbf..f88f4dd 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -169,19 +169,22 @@ public class MetaStoreUtils {
return true;
}
- public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh,
+ public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh,
boolean madeDir) throws MetaException {
- return updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir, false);
+ return updateTableStatsFast(db, tbl, wh, madeDir, false);
}
- public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh,
+ public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh,
boolean madeDir, boolean forceRecompute) throws MetaException {
- return updateUnpartitionedTableStatsFast(tbl,
- wh.getFileStatusesForUnpartitionedTable(db, tbl), madeDir, forceRecompute);
+ FileStatus[] fileStatuses = {};
+ if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned
+ fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl);
+ }
+ return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute);
}
/**
- * Updates the numFiles and totalSize parameters for the passed unpartitioned Table by querying
+ * Updates the numFiles and totalSize parameters for the passed Table by querying
* the warehouse if the passed Table does not already have values for these parameters.
* @param tbl
* @param fileStatus
@@ -190,7 +193,7 @@ public class MetaStoreUtils {
* these parameters set
* @return true if the stats were updated, false otherwise
*/
- public static boolean updateUnpartitionedTableStatsFast(Table tbl,
+ public static boolean updateTableStatsFast(Table tbl,
FileStatus[] fileStatus, boolean newDir, boolean forceRecompute) throws MetaException {
Map<String,String> params = tbl.getParameters();
http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index d20f196..51ff262 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -386,7 +386,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
}
org.apache.hadoop.hive.metastore.api.Table newtCopy = deepCopyAndLowerCaseTable(newt);
- MetaStoreUtils.updateUnpartitionedTableStatsFast(newtCopy,
+ MetaStoreUtils.updateTableStatsFast(newtCopy,
getWh().getFileStatusesForSD(newtCopy.getSd()), false, true);
Table newTable = new Table(newtCopy);
String newDbName = newTable.getDbName();
[37/50] [abbrv] hive git commit: HIVE-11513: AvroLazyObjectInspector
could handle empty data better (Swarnim Kulkarni, reviewed by Xuefu Zhang)
Posted by se...@apache.org.
HIVE-11513: AvroLazyObjectInspector could handle empty data better (Swarnim Kulkarni, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4a6ac92
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4a6ac92
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4a6ac92
Branch: refs/heads/llap
Commit: b4a6ac921e0a283107d9e852116d76ba097ccee5
Parents: a162674
Author: Swarnim Kulkarni <ku...@gmail.com>
Authored: Fri Aug 21 11:17:46 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 21 11:17:46 2015 -0700
----------------------------------------------------------------------
.../serde2/avro/AvroLazyObjectInspector.java | 19 +++----
.../avro/TestAvroLazyObjectInspector.java | 59 ++++++++++++++++++++
2 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4a6ac92/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
index 9fc9873..756f566 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
@@ -138,7 +138,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
if (rowField instanceof LazyStruct) {
- if (LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled() && rowField != null) {
LOG.debug("Deserializing struct [" + rowField.getClass() + "]");
}
@@ -166,7 +166,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug("Returning [" + rowField.toString() + "] for field [" + f.getFieldName() + "]");
+ LOG.debug("Returning [" + rowField + "] for field [" + f.getFieldName() + "]");
}
// Just return the object. We need no further operation on it
@@ -223,7 +223,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
byte[] data = ((LazyStruct) struct).getBytes();
AvroDeserializer deserializer = new AvroDeserializer();
- if (data == null) {
+ if (data == null || data.length == 0) {
return null;
}
@@ -239,6 +239,12 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable();
if (readerSchema == null) {
+ offset = schemaRetriever.getOffset();
+
+ if (data.length < offset) {
+ throw new IllegalArgumentException("Data size cannot be less than [" + offset
+ + "]. Found [" + data.length + "]");
+ }
rs = schemaRetriever.retrieveReaderSchema(data);
@@ -257,13 +263,6 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
}
// adjust the data bytes according to any possible offset that was provided
- offset = schemaRetriever.getOffset();
-
- if (data.length < offset) {
- throw new IllegalArgumentException("Data size cannot be less than [" + offset
- + "]. Found [" + data.length + "]");
- }
-
if (LOG.isDebugEnabled()) {
LOG.debug("Retrieved writer Schema: " + ws.toString());
LOG.debug("Retrieved reader Schema: " + rs.toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/b4a6ac92/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
new file mode 100644
index 0000000..2b7d513
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
@@ -0,0 +1,59 @@
+package org.apache.hadoop.hive.serde2.avro;
+
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class TestAvroLazyObjectInspector {
+
+ @Test
+ public void testEmptyData(){
+ List<String> fieldNames = new ArrayList<String>();
+ fieldNames.add("myField");
+
+ List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+ ois.add(LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0)));
+
+ AvroLazyObjectInspector aloi = new AvroLazyObjectInspector(fieldNames, ois, null, (byte)0, new Text(), false, false, (byte)0);
+ LazyStruct lazyStruct = new LazyStruct(LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(fieldNames, ois, (byte)0, new Text(), false, false, (byte)0));
+
+ ByteArrayRef byteArrayRef = new ByteArrayRef();
+ byteArrayRef.setData(new byte[0]); // set data to empty explicitly
+ lazyStruct.init(byteArrayRef, 0, 0);
+
+ assertNull(aloi.getStructFieldData(lazyStruct, new TestStructField()));
+ }
+
+ class TestStructField implements StructField {
+
+ @Override
+ public String getFieldName() {
+ return "testfield";
+ }
+
+ @Override
+ public ObjectInspector getFieldObjectInspector() {
+ return LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0));
+ }
+
+ @Override
+ public int getFieldID() {
+ return 0;
+ }
+
+ @Override
+ public String getFieldComment() {
+ return null;
+ }
+ }
+}
[17/50] [abbrv] hive git commit: HIVE-11549: Hide Hive configuration
from spark driver launching process (reviewed by Chao)
Posted by se...@apache.org.
HIVE-11549: Hide Hive configuration from spark driver launching process (reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe1efe52
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe1efe52
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe1efe52
Branch: refs/heads/llap
Commit: fe1efe520b807f24085d87fea8fc1c5ee0c9e44c
Parents: 2ccd061
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Fri Aug 14 15:30:52 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Fri Aug 14 15:30:52 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hive/spark/client/SparkClientImpl.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fe1efe52/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
----------------------------------------------------------------------
diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index e1e64a7..2546a46 100644
--- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -423,6 +423,10 @@ class SparkClientImpl implements SparkClient {
LOG.info("Running client driver with argv: {}", cmd);
ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd);
+ // Prevent hive configurations from being visible in Spark.
+ pb.environment().remove("HIVE_HOME");
+ pb.environment().remove("HIVE_CONF_DIR");
+
if (isTesting != null) {
pb.environment().put("SPARK_TESTING", isTesting);
}
[31/50] [abbrv] hive git commit: HIVE-11502: Map side aggregation is
extremely slow (Yongzhi Chen, reviewed by Chao Sun)
Posted by se...@apache.org.
HIVE-11502: Map side aggregation is extremely slow (Yongzhi Chen, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab03dc97
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab03dc97
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab03dc97
Branch: refs/heads/llap
Commit: ab03dc97b53ac6138de1be68fcb2f8b15aef400f
Parents: f8b0261
Author: Yongzhi Chen <yo...@hotmail.com>
Authored: Wed Aug 19 18:21:54 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Wed Aug 19 18:21:54 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/KeyWrapperFactory.java | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ab03dc97/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
index bf4ba7f..22bd951 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.util.Arrays;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.lazy.LazyDouble;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectsEqualComparer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -103,7 +104,22 @@ public class KeyWrapperFactory {
@Override
public void setHashKey() {
- hashcode = Arrays.hashCode(keys);
+ if (keys == null) {
+ hashcode = 0;
+ } else {
+ hashcode = 1;
+ for (Object element : keys) {
+ hashcode = 31 * hashcode;
+ if(element != null) {
+ if(element instanceof LazyDouble) {
+ long v = Double.doubleToLongBits(((LazyDouble)element).getWritableObject().get());
+ hashcode = hashcode + (int) (v ^ (v >>> 32));
+ } else {
+ hashcode = hashcode + element.hashCode();
+ }
+ }
+ }
+ }
}
@Override
[18/50] [abbrv] hive git commit: HIVE-11571: Fix Hive PTest2 logging
configuration (Gopal V reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11571: Fix Hive PTest2 logging configuration (Gopal V reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3071ce96
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3071ce96
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3071ce96
Branch: refs/heads/llap
Commit: 3071ce96b6b8635f668d0698c18a727bea1b1de1
Parents: fe1efe5
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 15:40:48 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 15:40:48 2015 -0700
----------------------------------------------------------------------
testutils/ptest2/src/main/resources/log4j2.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3071ce96/testutils/ptest2/src/main/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
index 6502ad1..42141b7 100644
--- a/testutils/ptest2/src/main/resources/log4j2.xml
+++ b/testutils/ptest2/src/main/resources/log4j2.xml
@@ -75,5 +75,6 @@
<Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
<AppenderRef ref="${sys:hive.ptest.root.logger}"/>
</Logger>
+ </Loggers>
</Configuration>
[35/50] [abbrv] hive git commit: HIVE-11607 : Export tables broken
for data > 32 MB (Ashutosh Chauhan via Sushanth Sowmyan, Sergio Pena)
Posted by se...@apache.org.
HIVE-11607 : Export tables broken for data > 32 MB (Ashutosh Chauhan via Sushanth Sowmyan, Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3664e28
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3664e28
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3664e28
Branch: refs/heads/llap
Commit: e3664e2899456ed926b028ceba8167c6775eb85d
Parents: d78bb7c
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Fri Aug 21 10:10:52 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Fri Aug 21 10:12:01 2015 -0700
----------------------------------------------------------------------
shims/0.20S/pom.xml | 8 +++-
.../hadoop/hive/shims/Hadoop20SShims.java | 35 ++++++++---------
shims/0.23/pom.xml | 21 ++++------
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 41 ++++++++------------
4 files changed, 46 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.20S/pom.xml
----------------------------------------------------------------------
diff --git a/shims/0.20S/pom.xml b/shims/0.20S/pom.xml
index 040136b..565dd5e 100644
--- a/shims/0.20S/pom.xml
+++ b/shims/0.20S/pom.xml
@@ -53,5 +53,11 @@
<version>${hadoop-20S.version}</version>
<optional>true</optional>
</dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-tools</artifactId>
+ <version>${hadoop-20S.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index a56309f..6c6ccbc 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.shims;
import java.io.IOException;
import java.lang.Override;
-import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URI;
@@ -27,6 +26,7 @@ import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -71,6 +71,8 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.distcp2.DistCp;
+import org.apache.hadoop.tools.distcp2.DistCpOptions;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
@@ -503,7 +505,7 @@ public class Hadoop20SShims extends HadoopShimsSecure {
}
public class Hadoop20SFileStatus implements HdfsFileStatus {
- private FileStatus fileStatus;
+ private final FileStatus fileStatus;
public Hadoop20SFileStatus(FileStatus fileStatus) {
this.fileStatus = fileStatus;
}
@@ -629,28 +631,33 @@ public class Hadoop20SShims extends HadoopShimsSecure {
*/
public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim {
- private KerberosName kerberosName;
+ private final KerberosName kerberosName;
public KerberosNameShim(String name) {
kerberosName = new KerberosName(name);
}
+ @Override
public String getDefaultRealm() {
return kerberosName.getDefaultRealm();
}
+ @Override
public String getServiceName() {
return kerberosName.getServiceName();
}
+ @Override
public String getHostName() {
return kerberosName.getHostName();
}
+ @Override
public String getRealm() {
return kerberosName.getRealm();
}
+ @Override
public String getShortName() throws IOException {
return kerberosName.getShortName();
}
@@ -663,27 +670,17 @@ public class Hadoop20SShims extends HadoopShimsSecure {
@Override
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
- int rc;
-
- // Creates the command-line parameters for distcp
- String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()};
+ DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
+ options.setSkipCRC(true);
+ options.setSyncFolder(true);
try {
- Class clazzDistCp = Class.forName("org.apache.hadoop.tools.distcp2");
- Constructor c = clazzDistCp.getConstructor();
- c.setAccessible(true);
- Tool distcp = (Tool)c.newInstance();
- distcp.setConf(conf);
- rc = distcp.run(params);
- } catch (ClassNotFoundException e) {
- throw new IOException("Cannot find DistCp class package: " + e.getMessage());
- } catch (NoSuchMethodException e) {
- throw new IOException("Cannot get DistCp constructor: " + e.getMessage());
+ DistCp distcp = new DistCp(conf, options);
+ distcp.execute();
+ return true;
} catch (Exception e) {
throw new IOException("Cannot execute DistCp process: " + e, e);
}
-
- return (0 == rc) ? true : false;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.23/pom.xml
----------------------------------------------------------------------
diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml
index 060dd17..2e16956 100644
--- a/shims/0.23/pom.xml
+++ b/shims/0.23/pom.xml
@@ -144,18 +144,11 @@
<optional>true</optional>
<type>test-jar</type>
</dependency>
- </dependencies>
-
- <profiles>
- <profile>
- <id>hadoop-2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <version>${hadoop-23.version}</version>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop-23.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index e5be8d6..29d0f13 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.shims;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
@@ -28,12 +27,12 @@ import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
@@ -64,7 +63,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
@@ -93,8 +91,9 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.test.MiniTezCluster;
@@ -121,7 +120,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
zcr = true;
} catch (ClassNotFoundException ce) {
}
-
+
if (zcr) {
// in-memory HDFS is only available after zcr
try {
@@ -576,11 +575,11 @@ public class Hadoop23Shims extends HadoopShimsSecure {
org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
try {
- java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
+ java.lang.reflect.Constructor<org.apache.hadoop.mapred.TaskAttemptContextImpl> construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
Reporter.class);
construct.setAccessible(true);
- newContext = (org.apache.hadoop.mapred.TaskAttemptContext) construct.newInstance(
+ newContext = construct.newInstance(
new JobConf(conf), taskId, progressable);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -913,6 +912,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
* Cannot add Override annotation since FileSystem.access() may not exist in
* the version of hadoop used to build Hive.
*/
+ @Override
public void access(Path path, FsAction action) throws AccessControlException,
FileNotFoundException, IOException {
Path underlyingFsPath = swizzleParamPath(path);
@@ -1179,7 +1179,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
}
}
-
+
@Override
public HadoopShims.StoragePolicyShim getStoragePolicyShim(FileSystem fs) {
@@ -1195,27 +1195,17 @@ public class Hadoop23Shims extends HadoopShimsSecure {
@Override
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
- int rc;
-
- // Creates the command-line parameters for distcp
- String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()};
+ DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
+ options.setSkipCRC(true);
+ options.setSyncFolder(true);
try {
- Class clazzDistCp = Class.forName("org.apache.hadoop.tools.DistCp");
- Constructor c = clazzDistCp.getConstructor();
- c.setAccessible(true);
- Tool distcp = (Tool)c.newInstance();
- distcp.setConf(conf);
- rc = distcp.run(params);
- } catch (ClassNotFoundException e) {
- throw new IOException("Cannot find DistCp class package: " + e.getMessage());
- } catch (NoSuchMethodException e) {
- throw new IOException("Cannot get DistCp constructor: " + e.getMessage());
+ DistCp distcp = new DistCp(conf, options);
+ distcp.execute();
+ return true;
} catch (Exception e) {
throw new IOException("Cannot execute DistCp process: " + e, e);
}
-
- return (0 == rc);
}
private static Boolean hdfsEncryptionSupport;
@@ -1250,7 +1240,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
*/
private KeyProvider keyProvider = null;
- private Configuration conf;
+ private final Configuration conf;
public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException {
DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf);
@@ -1409,6 +1399,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
return result;
}
+ @Override
public void addDelegationTokens(FileSystem fs, Credentials cred, String uname) throws IOException {
// Use method addDelegationTokens instead of getDelegationToken to get all the tokens including KMS.
fs.addDelegationTokens(uname, cred);
[22/50] [abbrv] hive git commit: HIVE-11490 : Lazily call
ASTNode::toStringTree() after tree modification (Hari Subramaniyan,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11490 : Lazily call ASTNode::toStringTree() after tree modification (Hari Subramaniyan, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d8515df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d8515df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d8515df
Branch: refs/heads/llap
Commit: 9d8515df6ff722cd81b9b42a582c422adeac8849
Parents: 97a6cd3
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Mon Aug 17 11:32:06 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Mon Aug 17 11:32:06 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/lib/DefaultGraphWalker.java | 2 +-
.../org/apache/hadoop/hive/ql/parse/ASTNode.java | 18 +++++++++---------
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9d8515df/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
index cf9131d..583c113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
@@ -108,7 +108,7 @@ public class DefaultGraphWalker implements GraphWalker {
while (toWalk.size() > 0) {
Node nd = toWalk.remove(0);
walk(nd);
- if (nodeOutput != null) {
+ if (nodeOutput != null && getDispatchedList().contains(nd)) {
nodeOutput.put(nd, retMap.get(nd));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9d8515df/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
index 136d481..b96e2eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
@@ -143,9 +143,11 @@ public class ASTNode extends CommonTree implements Node,Serializable {
retNode = (ASTNode) retNode.parent;
}
rootNode=retNode;
- rootNode.astStr = new StringBuffer();
- rootNode.toStringTree(rootNode);
- rootNode.isValidASTStr = true;
+ if (!rootNode.isValidASTStr) {
+ rootNode.astStr = new StringBuffer();
+ rootNode.toStringTree(rootNode);
+ rootNode.isValidASTStr = true;
+ }
return retNode;
}
@@ -159,9 +161,6 @@ public class ASTNode extends CommonTree implements Node,Serializable {
rootNode.astStr = null;
rootNode.isValidASTStr = false;
}
- // The root might have changed because of tree modifications.
- // Compute the new root for this tree and set the astStr.
- getRootNodeWithValidASTStr(false);
}
private int getMemoizedStringLen() {
@@ -225,9 +224,10 @@ public class ASTNode extends CommonTree implements Node,Serializable {
@Override
public String toStringTree() {
- // The tree modifier functions invalidate the old astStr, rootNode, etc.
- // Hence, we can use the memoized root node and string values here.
- ASTNode rootNode = (ASTNode)this.getRootNodeWithValidASTStr(true);
+
+ // The root might have changed because of tree modifications.
+ // Compute the new root for this tree and set the astStr.
+ getRootNodeWithValidASTStr(true);
// If rootNotModified is false, then startIndx and endIndx will be stale.
if (startIndx >= 0 && endIndx <= rootNode.getMemoizedStringLen()) {
[19/50] [abbrv] hive git commit: HIVE-11556: HiveFilter.copy should
take the condition given as a parameter (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11556: HiveFilter.copy should take the condition given as a parameter (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/147347a3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/147347a3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/147347a3
Branch: refs/heads/llap
Commit: 147347a31dd021ac3a6f956c7c02a77b505bce7b
Parents: 3071ce9
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Sat Aug 15 10:49:55 2015 +0300
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Sat Aug 15 10:49:55 2015 +0300
----------------------------------------------------------------------
.../hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/147347a3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
index de61e48..eb97bec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
@@ -39,7 +39,7 @@ public class HiveFilter extends Filter implements HiveRelNode {
@Override
public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
- return new HiveFilter(getCluster(), traitSet, input, getCondition());
+ return new HiveFilter(getCluster(), traitSet, input, condition);
}
@Override
[04/50] [abbrv] hive git commit: HIVE-11467 : WriteBuffers rounding
wbSize to next power of 2 may cause OOM (Wei Zheng,
reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11467 : WriteBuffers rounding wbSize to next power of 2 may cause OOM (Wei Zheng, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/433ea9cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/433ea9cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/433ea9cd
Branch: refs/heads/llap
Commit: 433ea9cda59a986689cf99953826512e4ff07e4d
Parents: 1a75644
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Aug 13 14:46:07 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Aug 13 14:46:07 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/persistence/HybridHashTableContainer.java | 4 ++++
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
index ad1246d..ff64f52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
@@ -290,6 +290,10 @@ public class HybridHashTableContainer
}
}
+ // Round to power of 2 here, as is required by WriteBuffers
+ writeBufferSize = Integer.bitCount(writeBufferSize) == 1 ?
+ writeBufferSize : Integer.highestOneBit(writeBufferSize);
+
// Cap WriteBufferSize to avoid large preallocations
writeBufferSize = writeBufferSize < minWbSize ? minWbSize : Math.min(maxWbSize, writeBufferSize);
LOG.info("Write buffer size: " + writeBufferSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
index acb51f9..05d9359 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
@@ -56,7 +56,7 @@ public final class WriteBuffers implements RandomAccessOutput {
public WriteBuffers(int wbSize, long maxSize) {
- this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : (Integer.highestOneBit(wbSize) << 1);
+ this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : Integer.highestOneBit(wbSize);
this.wbSizeLog2 = 31 - Integer.numberOfLeadingZeros(this.wbSize);
this.offsetMask = this.wbSize - 1;
this.maxSize = maxSize;
[26/50] [abbrv] hive git commit: HIVE-11580:
ThriftUnionObjectInspector#toString throws NPE (Jimmy, reviewed by Chao)
Posted by se...@apache.org.
HIVE-11580: ThriftUnionObjectInspector#toString throws NPE (Jimmy, reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6433c3b0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6433c3b0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6433c3b0
Branch: refs/heads/llap
Commit: 6433c3b0b8e305c8dddc6ea2fe94cfcd5062e40d
Parents: 5edbf31
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Fri Aug 14 15:48:23 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Tue Aug 18 08:49:21 2015 -0700
----------------------------------------------------------------------
.../ThriftUnionObjectInspector.java | 26 ++++++++++----------
.../TestThriftObjectInspectors.java | 10 ++++++--
2 files changed, 21 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6433c3b0/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
index 17add28..600abbb 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
@@ -18,18 +18,18 @@
package org.apache.hadoop.hive.serde2.objectinspector;
-import com.google.common.primitives.UnsignedBytes;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.thrift.TFieldIdEnum;
-import org.apache.thrift.TUnion;
-import org.apache.thrift.meta_data.FieldMetaData;
-
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.thrift.TFieldIdEnum;
+import org.apache.thrift.TUnion;
+import org.apache.thrift.meta_data.FieldMetaData;
+
+import com.google.common.primitives.UnsignedBytes;
+
/**
* Always use the ObjectInspectorFactory to create new ObjectInspector objects,
* instead of directly creating an instance of this class.
@@ -37,7 +37,8 @@ import java.util.Map;
public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector implements UnionObjectInspector {
private static final String FIELD_METADATA_MAP = "metaDataMap";
- private List<ObjectInspector> ois;
+ private List<ObjectInspector> ois;
+ private List<StandardStructObjectInspector.MyField> fields;
@Override
public boolean shouldIgnoreField(String name) {
@@ -88,10 +89,14 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
try {
final Map<? extends TFieldIdEnum, FieldMetaData> fieldMap = (Map<? extends TFieldIdEnum, FieldMetaData>) fieldMetaData.get(null);
+ fields = new ArrayList<StandardStructObjectInspector.MyField>(fieldMap.size());
this.ois = new ArrayList<ObjectInspector>();
for(Map.Entry<? extends TFieldIdEnum, FieldMetaData> metadata : fieldMap.entrySet()) {
- final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, metadata.getValue().fieldName);
+ int fieldId = metadata.getKey().getThriftFieldId();
+ String fieldName = metadata.getValue().fieldName;
+ final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, fieldName);
final ObjectInspector reflectionObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(fieldType, options);
+ fields.add(new StandardStructObjectInspector.MyField(fieldId, fieldName, reflectionObjectInspector));
this.ois.add(reflectionObjectInspector);
}
} catch (IllegalAccessException e) {
@@ -112,10 +117,5 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
public String getTypeName() {
return ObjectInspectorUtils.getStandardUnionTypeName(this);
}
-
- @Override
- public Object create() {
- return ReflectionUtils.newInstance(objectClass, null);
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6433c3b0/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
index 968cd22..85f2bd6 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
@@ -23,14 +23,15 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.thrift.test.Complex;
import org.apache.hadoop.hive.serde2.thrift.test.IntString;
+import org.apache.hadoop.hive.serde2.thrift.test.PropValueUnion;
import org.apache.hadoop.hive.serde2.thrift.test.SetIntString;
+import junit.framework.TestCase;
+
/**
* TestThriftObjectInspectors.
*
@@ -60,6 +61,11 @@ public class TestThriftObjectInspectors extends TestCase {
assertNull(soi.getStructFieldData(null, fields.get(i)));
}
+ ObjectInspector oi = ObjectInspectorFactory
+ .getReflectionObjectInspector(PropValueUnion.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
+ assertNotNull(oi.toString());
+
// real object
Complex c = new Complex();
c.setAint(1);
[10/50] [abbrv] hive git commit: HIVE-11304: Migrate to Log4j2 from
Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11304: Migrate to Log4j2 from Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c93d6c77
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c93d6c77
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c93d6c77
Branch: refs/heads/llap
Commit: c93d6c77e31e2eb9b40f5167ab3491d44eae351a
Parents: a4849cb
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 10:17:20 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 10:17:20 2015 -0700
----------------------------------------------------------------------
accumulo-handler/pom.xml | 4 -
.../src/main/resources/beeline-log4j.properties | 24 --
beeline/src/main/resources/beeline-log4j2.xml | 40 ++++
bin/ext/beeline.sh | 2 +-
bin/hive | 3 +
common/pom.xml | 22 +-
.../org/apache/hadoop/hive/common/LogUtils.java | 18 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 8 +-
common/src/main/resources/hive-log4j.properties | 88 --------
common/src/main/resources/hive-log4j2.xml | 111 +++++++++
.../hadoop/hive/conf/TestHiveLogging.java | 8 +-
.../resources/hive-exec-log4j-test.properties | 59 -----
.../test/resources/hive-exec-log4j2-test.xml | 86 +++++++
.../test/resources/hive-log4j-test.properties | 71 ------
common/src/test/resources/hive-log4j2-test.xml | 95 ++++++++
data/conf/hive-log4j-old.properties | 82 -------
data/conf/hive-log4j.properties | 97 --------
data/conf/hive-log4j2.xml | 148 ++++++++++++
data/conf/spark/log4j.properties | 24 --
data/conf/spark/log4j2.xml | 74 ++++++
docs/xdocs/language_manual/cli.xml | 2 +-
hcatalog/bin/hcat_server.sh | 2 +-
hcatalog/bin/templeton.cmd | 4 +-
hcatalog/scripts/hcat_server_start.sh | 2 +-
.../content/xdocs/configuration.xml | 2 +-
.../src/documentation/content/xdocs/install.xml | 2 +-
.../deployers/config/hive/hive-log4j.properties | 88 --------
.../deployers/config/hive/hive-log4j2.xml | 111 +++++++++
.../templeton/deployers/start_hive_services.sh | 2 +-
.../webhcat/svr/src/main/bin/webhcat_server.sh | 4 +-
.../src/main/config/webhcat-log4j.properties | 45 ----
.../svr/src/main/config/webhcat-log4j2.xml | 75 +++++++
.../main/java/org/apache/hive/hplsql/Exec.java | 2 +
.../operation/TestOperationLoggingLayout.java | 136 +++++++++++
itests/pom.xml | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../metastore/txn/TestCompactionTxnHandler.java | 40 +++-
.../hive/metastore/txn/TestTxnHandler.java | 66 ++++--
packaging/src/main/assembly/bin.xml | 14 +-
pom.xml | 37 ++-
ql/pom.xml | 17 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 29 ++-
.../hive/ql/exec/mr/HadoopJobExecHelper.java | 20 +-
.../ql/io/rcfile/stats/PartialScanTask.java | 20 +-
.../hadoop/hive/ql/log/HiveEventCounter.java | 135 +++++++++++
.../apache/hadoop/hive/ql/log/NullAppender.java | 63 ++++++
.../ql/log/PidDailyRollingFileAppender.java | 33 ---
.../hive/ql/log/PidFilePatternConverter.java | 62 ++++++
.../main/resources/hive-exec-log4j.properties | 77 -------
ql/src/main/resources/hive-exec-log4j2.xml | 110 +++++++++
.../hadoop/hive/ql/log/TestLog4j2Appenders.java | 95 ++++++++
.../hadoop/hive/ql/metadata/StringAppender.java | 128 +++++++++++
.../hadoop/hive/ql/metadata/TestHive.java | 50 +++--
.../hive/service/cli/CLIServiceUtils.java | 7 -
.../cli/operation/LogDivertAppender.java | 223 +++++++++++--------
.../service/cli/operation/OperationManager.java | 17 +-
shims/common/pom.xml | 17 +-
.../hadoop/hive/shims/HiveEventCounter.java | 102 ---------
.../src/test/resources/log4j.properties | 23 --
spark-client/src/test/resources/log4j2.xml | 39 ++++
storage-api/pom.xml | 7 -
testutils/ptest2/pom.xml | 20 ++
.../ptest2/src/main/resources/log4j.properties | 37 ---
testutils/ptest2/src/main/resources/log4j2.xml | 79 +++++++
64 files changed, 1989 insertions(+), 1123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/accumulo-handler/pom.xml
----------------------------------------------------------------------
diff --git a/accumulo-handler/pom.xml b/accumulo-handler/pom.xml
index ee40f72..4e3a087 100644
--- a/accumulo-handler/pom.xml
+++ b/accumulo-handler/pom.xml
@@ -91,10 +91,6 @@
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/beeline/src/main/resources/beeline-log4j.properties
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/beeline-log4j.properties b/beeline/src/main/resources/beeline-log4j.properties
deleted file mode 100644
index fe47d94..0000000
--- a/beeline/src/main/resources/beeline-log4j.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=WARN, console
-
-######## console appender ########
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
-log4j.appender.console.encoding=UTF-8
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/beeline/src/main/resources/beeline-log4j2.xml
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/beeline-log4j2.xml b/beeline/src/main/resources/beeline-log4j2.xml
new file mode 100644
index 0000000..5f09741
--- /dev/null
+++ b/beeline/src/main/resources/beeline-log4j2.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="BeelineLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">WARN</Property>
+ <Property name="hive.root.logger">console</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ </Root>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/bin/ext/beeline.sh
----------------------------------------------------------------------
diff --git a/bin/ext/beeline.sh b/bin/ext/beeline.sh
index ab3dc1a..9de8f6c 100644
--- a/bin/ext/beeline.sh
+++ b/bin/ext/beeline.sh
@@ -31,7 +31,7 @@ beeline () {
hadoopClasspath="${HADOOP_CLASSPATH}:"
fi
export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}"
- export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configuration=beeline-log4j.properties "
+ export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.xml "
exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@"
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/bin/hive
----------------------------------------------------------------------
diff --git a/bin/hive b/bin/hive
index 5dc93fb..ad7139e 100755
--- a/bin/hive
+++ b/bin/hive
@@ -193,6 +193,9 @@ if [ "$HADOOP_HOME" == "" ]; then
exit 4;
fi
+# to avoid errors from log4j2 automatic configuration loading
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.xml "
+
HADOOP=$HADOOP_HOME/bin/hadoop
if [ ! -f ${HADOOP} ]; then
echo "Cannot find hadoop installation: \$HADOOP_HOME or \$HADOOP_PREFIX must be set or hadoop must be in the path";
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index a7997e2..dba814d 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -66,14 +66,24 @@
<version>${joda.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
index 9118675..3ca5c0f 100644
--- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
@@ -18,26 +18,22 @@
package org.apache.hadoop.hive.common;
-import java.net.URL;
import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
+import java.net.URL;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.PropertyConfigurator;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.logging.log4j.core.config.Configurator;
/**
* Utilities common to logging operations.
*/
public class LogUtils {
- private static final String HIVE_L4J = "hive-log4j.properties";
- private static final String HIVE_EXEC_L4J = "hive-exec-log4j.properties";
+ private static final String HIVE_L4J = "hive-log4j2.xml";
+ private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.xml";
private static final Log l4j = LogFactory.getLog(LogUtils.class);
@SuppressWarnings("serial")
@@ -95,8 +91,7 @@ public class LogUtils {
}
System.setProperty(HiveConf.ConfVars.HIVEQUERYID.toString(), queryId);
}
- LogManager.resetConfiguration();
- PropertyConfigurator.configure(log4jFileName);
+ Configurator.initialize(null, log4jFileName);
logConfigLocation(conf);
return ("Logging initialized using configuration in " + log4jConfigFile);
}
@@ -123,8 +118,7 @@ public class LogUtils {
break;
}
if (hive_l4j != null) {
- LogManager.resetConfiguration();
- PropertyConfigurator.configure(hive_l4j);
+ Configurator.initialize(null, hive_l4j.toString());
logConfigLocation(conf);
return (logMessage + "\n" + "Logging initialized using configuration in " + hive_l4j);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 11b9f78..730f5be 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1696,13 +1696,13 @@ public class HiveConf extends Configuration {
// logging configuration
HIVE_LOG4J_FILE("hive.log4j.file", "",
"Hive log4j configuration file.\n" +
- "If the property is not set, then logging will be initialized using hive-log4j.properties found on the classpath.\n" +
- "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
+ "If the property is not set, then logging will be initialized using hive-log4j2.xml found on the classpath.\n" +
+ "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" +
"which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),
HIVE_EXEC_LOG4J_FILE("hive.exec.log4j.file", "",
"Hive log4j configuration file for execution mode(sub command).\n" +
- "If the property is not set, then logging will be initialized using hive-exec-log4j.properties found on the classpath.\n" +
- "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
+ "If the property is not set, then logging will be initialized using hive-exec-log4j2.xml found on the classpath.\n" +
+ "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" +
"which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),
HIVE_LOG_EXPLAIN_OUTPUT("hive.log.explain.output", false,
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/main/resources/hive-log4j.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/hive-log4j.properties b/common/src/main/resources/hive-log4j.properties
deleted file mode 100644
index 14fa725..0000000
--- a/common/src/main/resources/hive-log4j.properties
+++ /dev/null
@@ -1,88 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=INFO,DRFA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=${hive.log.threshold}
-
-#
-# Daily Rolling File Appender
-#
-# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
-# for different CLI session.
-#
-# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
-log4j.appender.console.encoding=UTF-8
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/main/resources/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/common/src/main/resources/hive-log4j2.xml b/common/src/main/resources/hive-log4j2.xml
new file mode 100644
index 0000000..31b8fcc
--- /dev/null
+++ b/common/src/main/resources/hive-log4j2.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">DRFA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.log.file">hive.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File> -->
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
index d5cedb1..92269e7 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
@@ -21,12 +21,12 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hive.common.util.HiveTestUtils;
+import junit.framework.TestCase;
+
/**
* TestHiveLogging
*
@@ -104,9 +104,9 @@ public class TestHiveLogging extends TestCase {
// customized log4j config log file to be: /${test.tmp.dir}/TestHiveLogging/hiveLog4jTest.log
File customLogPath = new File(new File(System.getProperty("test.tmp.dir")),
System.getProperty("user.name") + "-TestHiveLogging/");
- String customLogName = "hiveLog4jTest.log";
+ String customLogName = "hiveLog4j2Test.log";
File customLogFile = new File(customLogPath, customLogName);
RunTest(customLogFile,
- "hive-log4j-test.properties", "hive-exec-log4j-test.properties");
+ "hive-log4j2-test.xml", "hive-exec-log4j2-test.xml");
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-exec-log4j-test.properties
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-exec-log4j-test.properties b/common/src/test/resources/hive-exec-log4j-test.properties
deleted file mode 100644
index 1e53f26..0000000
--- a/common/src/test/resources/hive-exec-log4j-test.properties
+++ /dev/null
@@ -1,59 +0,0 @@
-# Define some default values that can be overridden by system properties
-hive.root.logger=INFO,FA
-hive.log.dir=/${test.tmp.dir}/${user.name}-TestHiveLogging
-hive.log.file=hiveExecLog4jTest.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=WARN
-
-#
-# File Appender
-#
-
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
-
-
-log4j.category.DataNucleus=ERROR,FA
-log4j.category.Datastore=ERROR,FA
-log4j.category.Datastore.Schema=ERROR,FA
-log4j.category.JPOX.Datastore=ERROR,FA
-log4j.category.JPOX.Plugin=ERROR,FA
-log4j.category.JPOX.MetaData=ERROR,FA
-log4j.category.JPOX.Query=ERROR,FA
-log4j.category.JPOX.General=ERROR,FA
-log4j.category.JPOX.Enhancer=ERROR,FA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-exec-log4j2-test.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-exec-log4j2-test.xml b/common/src/test/resources/hive-exec-log4j2-test.xml
new file mode 100644
index 0000000..b5f2cb4
--- /dev/null
+++ b/common/src/test/resources/hive-exec-log4j2-test.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveExecLog4j2Test"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">FA</Property>
+ <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
+ <Property name="hive.log.file">hiveExecLog4j2Test.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n" />
+ </File>
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <!-- Silence useless ZK logs -->
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-log4j-test.properties
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-log4j-test.properties b/common/src/test/resources/hive-log4j-test.properties
deleted file mode 100644
index 0348325..0000000
--- a/common/src/test/resources/hive-log4j-test.properties
+++ /dev/null
@@ -1,71 +0,0 @@
-# Define some default values that can be overridden by system properties
-hive.root.logger=WARN,DRFA
-hive.log.dir=${test.tmp.dir}/${user.name}-TestHiveLogging
-hive.log.file=hiveLog4jTest.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=WARN
-
-#
-# Daily Rolling File Appender
-#
-# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
-# for different CLI session.
-#
-# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.console.encoding=UTF-8
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-log4j2-test.xml
----------------------------------------------------------------------
diff --git a/common/src/test/resources/hive-log4j2-test.xml b/common/src/test/resources/hive-log4j2-test.xml
new file mode 100644
index 0000000..63b46c8
--- /dev/null
+++ b/common/src/test/resources/hive-log4j2-test.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveLog4j2Test"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">WARN</Property>
+ <Property name="hive.root.logger">DRFA</Property>
+ <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
+ <Property name="hive.log.file">hiveLog4j2Test.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Daily Rolling File Appender -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <!-- Silence useless ZK logs -->
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j-old.properties
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j-old.properties b/data/conf/hive-log4j-old.properties
deleted file mode 100644
index f274b8c..0000000
--- a/data/conf/hive-log4j-old.properties
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.root.logger=DEBUG,DRFA
-hive.log.dir=${build.dir.hive}/ql/tmp/
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=WARN
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j.properties
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties
deleted file mode 100644
index 023e3c2..0000000
--- a/data/conf/hive-log4j.properties
+++ /dev/null
@@ -1,97 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.root.logger=DEBUG,DRFA
-hive.log.dir=${test.tmp.dir}/log/
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=WARN
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-# Warning: If you enable the following appender it will fill up your disk if you don't have a cleanup job!
-# This uses the updated rolling file appender from log4j-extras that supports a reliable time-based rolling policy.
-# See http://logging.apache.org/log4j/companions/extras/apidocs/org/apache/log4j/rolling/TimeBasedRollingPolicy.html
-# Add "DAILY" to hive.root.logger above if you want to use this.
-log4j.appender.DAILY=org.apache.log4j.rolling.RollingFileAppender
-log4j.appender.DAILY.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy
-log4j.appender.DAILY.rollingPolicy.ActiveFileName=${hive.log.dir}/${hive.log.file}
-log4j.appender.DAILY.rollingPolicy.FileNamePattern=${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
-log4j.appender.DAILY.layout=org.apache.log4j.PatternLayout
-log4j.appender.DAILY.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
-log4j.logger.org.apache.zookeeper=INFO,DRFA
-log4j.logger.org.apache.zookeeper.server.ServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocket=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
-log4j.logger.org.apache.hadoop.hive.ql.log.PerfLogger=${hive.ql.log.PerfLogger.level}
-log4j.logger.org.apache.hadoop.hive.ql.exec.Operator=INFO,DRFA
-log4j.logger.org.apache.hadoop.hive.serde2.lazy=INFO,DRFA
-log4j.logger.org.apache.hadoop.hive.metastore.ObjectStore=INFO,DRFA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
new file mode 100644
index 0000000..c9adfa2
--- /dev/null
+++ b/data/conf/hive-log4j2.xml
@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">DEBUG</Property>
+ <Property name="hive.root.logger">DRFA</Property>
+ <Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>
+ <Property name="hive.log.file">hive.log</Property>
+ <Property name="hive.ql.log.PerfLogger.level">INFO</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File> -->
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="{sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.hadoop.conf.Configuration" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.server.ServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocket" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hive.serde2.lazy" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.hive.metastore.ObjectStore" level="INFO">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/spark/log4j.properties
----------------------------------------------------------------------
diff --git a/data/conf/spark/log4j.properties b/data/conf/spark/log4j.properties
deleted file mode 100644
index 8838c0e..0000000
--- a/data/conf/spark/log4j.properties
+++ /dev/null
@@ -1,24 +0,0 @@
-log4j.rootCategory=DEBUG, DRFA
-
-hive.spark.log.dir=target/tmp/log
-# Settings to quiet third party logs that are too verbose
-log4j.logger.org.eclipse.jetty=WARN
-log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
-log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
-log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.DRFA.File=${hive.spark.log.dir}/spark.log
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/spark/log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/log4j2.xml b/data/conf/spark/log4j2.xml
new file mode 100644
index 0000000..395a2bf
--- /dev/null
+++ b/data/conf/spark/log4j2.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="SparkLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="spark.log.level">DEBUG</Property>
+ <Property name="spark.root.logger">DRFA</Property>
+ <Property name="spark.log.dir">target/tmp/log</Property>
+ <Property name="spark.log.file">spark.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <File name="FA" fileName="${sys:spark.log.dir}/${sys:spark.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File> -->
+
+ <!-- Daily Rolling File Appender -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:spark.log.dir}/${sys:spark.log.file}"
+ filePattern="${sys:spark.log.dir}/${sys:spark.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ </Appenders>
+
+ <Loggers>
+ <Root level="DEBUG">
+ <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
+ </Root>
+
+ <Logger name="org.apache.spark.repl.SparkIMain$exprTyper" level="INFO">
+ <AppenderRef ref="${sys:spark.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.spark.repl.SparkILoop$SparkILoopInterpreter" level="INFO">
+ <AppenderRef ref="${sys:spark.root.logger}"/>
+ </Logger>
+ <Logger name="org.eclipse.jetty" level="WARN">
+ <AppenderRef ref="${sys:spark.root.logger}"/>
+ </Logger>
+ <Logger name="org.eclipse.jetty.util.component.AbstractLifeCycle" level="ERROR">
+ <AppenderRef ref="${sys:spark.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/docs/xdocs/language_manual/cli.xml
----------------------------------------------------------------------
diff --git a/docs/xdocs/language_manual/cli.xml b/docs/xdocs/language_manual/cli.xml
index a293382..eb91e44 100644
--- a/docs/xdocs/language_manual/cli.xml
+++ b/docs/xdocs/language_manual/cli.xml
@@ -163,7 +163,7 @@ Sample Usage:
<section name="Logging" href="logging">
<p>
-Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use <i>hive-log4j.default</i> in the <i>conf/</i> directory of the hive installation which writes out logs to <i>/tmp/$USER/hive.log</i> and uses the <i>WARN</i> level.
+Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use <i>hive-log4j2.xml</i> in the <i>conf/</i> directory of the hive installation which writes out logs to <i>/tmp/$USER/hive.log</i> and uses the <i>WARN</i> level.
</p>
<p>
It is often desirable to emit the logs to the standard output and/or change the logging level for debugging purposes. These can be done from the command line as follows: </p>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/bin/hcat_server.sh
----------------------------------------------------------------------
diff --git a/hcatalog/bin/hcat_server.sh b/hcatalog/bin/hcat_server.sh
index 6b09d3e..d1aecb8 100644
--- a/hcatalog/bin/hcat_server.sh
+++ b/hcatalog/bin/hcat_server.sh
@@ -84,7 +84,7 @@ function start_hcat() {
export AUX_CLASSPATH=${AUX_CLASSPATH}
export HADOOP_HOME=$HADOOP_HOME
- #export HADOOP_OPTS="-Dlog4j.configuration=file://${HCAT_PREFIX}/conf/log4j.properties"
+ #export HADOOP_OPTS="-Dlog4j.configurationFile=file://${HCAT_PREFIX}/conf/log4j2.xml"
export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HCAT_LOG_DIR}/hcat_err_pid%p.log -Xloggc:${HCAT_LOG_DIR}/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-2048} # 8G is better if you have it
export METASTORE_PORT=${METASTORE_PORT:-9083}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/bin/templeton.cmd
----------------------------------------------------------------------
diff --git a/hcatalog/bin/templeton.cmd b/hcatalog/bin/templeton.cmd
index e9a735d..759f654 100644
--- a/hcatalog/bin/templeton.cmd
+++ b/hcatalog/bin/templeton.cmd
@@ -59,9 +59,9 @@ setlocal enabledelayedexpansion
if not defined TEMPLETON_LOG4J (
@rem must be prefixed with file: otherwise config is not picked up
- set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j.properties
+ set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j2.xml
)
- set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configuration=%TEMPLETON_LOG4J% %HADOOP_OPTS%
+ set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configurationFile=%TEMPLETON_LOG4J% %HADOOP_OPTS%
set arguments=%JAVA_HEAP_MAX% %TEMPLETON_OPTS% -classpath %CLASSPATH% org.apache.hive.hcatalog.templeton.Main
if defined service_entry (
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/scripts/hcat_server_start.sh
----------------------------------------------------------------------
diff --git a/hcatalog/scripts/hcat_server_start.sh b/hcatalog/scripts/hcat_server_start.sh
index 1670b70..872d1b5 100755
--- a/hcatalog/scripts/hcat_server_start.sh
+++ b/hcatalog/scripts/hcat_server_start.sh
@@ -70,7 +70,7 @@ export AUX_CLASSPATH=${AUX_CLASSPATH}
export HADOOP_HOME=$HADOOP_HOME
-#export HADOOP_OPTS="-Dlog4j.configuration=file://${ROOT}/conf/log4j.properties"
+#export HADOOP_OPTS="-Dlog4j.configurationFile=file://${ROOT}/conf/log4j2.xml"
export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${ROOT}/var/log/hcat_err_pid%p.log -Xloggc:${ROOT}/var/log/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
export HADOOP_HEAPSIZE=2048 # 8G is better if you have it
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
----------------------------------------------------------------------
diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
index 9757b9c..6385e40 100644
--- a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
+++ b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
@@ -66,7 +66,7 @@ ${env.PIG_HOME}/bin/pig
uncertainty.</p>
<p><strong>Note:</strong> The location of the log files created by Templeton and some other properties
- of the logging system are set in the webhcat-log4j.properties file.</p>
+ of the logging system are set in the webhcat-log4j2.xml file.</p>
<section>
<title>Variables</title>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
----------------------------------------------------------------------
diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
index 16da248..e2953a9 100644
--- a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
+++ b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
@@ -241,7 +241,7 @@
<p>Server activity logs are located in
<em>root</em><code>/var/log/hcat_server</code>. Logging configuration is located at
- <em>root</em><code>/conf/log4j.properties</code>. Server logging uses
+ <em>root</em><code>/conf/log4j2.xml</code>. Server logging uses
<code>DailyRollingFileAppender</code> by default. It will generate a new
file per day and does not expire old log files automatically.</p>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
deleted file mode 100644
index 82684b3..0000000
--- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
+++ /dev/null
@@ -1,88 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=DEBUG,DRFA
-hive.log.dir=/tmp/ekoifman
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=${hive.log.threshold}
-
-#
-# Daily Rolling File Appender
-#
-# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
-# for different CLI session.
-#
-# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
-log4j.appender.console.encoding=UTF-8
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
new file mode 100644
index 0000000..30f7603
--- /dev/null
+++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HCatE2ELog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">DEBUG</Property>
+ <Property name="hive.root.logger">DRFA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.log.file">hive.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File> -->
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
index 8cc9353..e59177c 100755
--- a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
+++ b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
@@ -31,7 +31,7 @@ cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.
#cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml ${HIVE_HOME}/conf/hive-site.xml
cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
-cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties ${HIVE_HOME}/conf/hive-log4j.properties
+cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml ${HIVE_HOME}/conf/hive-log4j2.xml
if [ -f ${MYSQL_CLIENT_JAR} ]; then
cp ${MYSQL_CLIENT_JAR} ${HIVE_HOME}/lib
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
index 0be8dde..c80fdd5 100644
--- a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
+++ b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
@@ -215,11 +215,11 @@ else
fi
if [[ -z "$WEBHCAT_LOG4J" ]]; then
- WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j.properties";
+ WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j2.xml";
fi
export HADOOP_USER_CLASSPATH_FIRST=true
-export HADOOP_OPTS="${HADOOP_OPTS} -Dwebhcat.log.dir=$WEBHCAT_LOG_DIR -Dlog4j.configuration=$WEBHCAT_LOG4J"
+export HADOOP_OPTS="${HADOOP_OPTS} -Dwebhcat.log.dir=$WEBHCAT_LOG_DIR -Dlog4j.configurationFile=$WEBHCAT_LOG4J"
start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR org.apache.hive.hcatalog.templeton.Main "
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
deleted file mode 100644
index 866052c..0000000
--- a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Define some default values that can be overridden by system properties
-webhcat.root.logger = INFO, standard
-webhcat.log.dir = .
-webhcat.log.file = webhcat.log
-
-log4j.rootLogger = ${webhcat.root.logger}
-
-# Logging Threshold
-log4j.threshhold = DEBUG
-
-log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
-log4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern = .yyyy-MM-dd
-
-log4j.appender.DRFA.layout = org.apache.log4j.PatternLayout
-
-log4j.appender.standard.layout = org.apache.log4j.PatternLayout
-log4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n
-
-# Class logging settings
-log4j.logger.com.sun.jersey = DEBUG
-log4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR
-log4j.logger.org.apache.hadoop = INFO
-log4j.logger.org.apache.hadoop.conf = WARN
-log4j.logger.org.apache.zookeeper = WARN
-log4j.logger.org.eclipse.jetty = INFO
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
new file mode 100644
index 0000000..40da974
--- /dev/null
+++ b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="WebhcatLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="webhcat.log.threshold">ALL</Property>
+ <Property name="webhcat.log.level">INFO</Property>
+ <Property name="webhcat.root.logger">standard</Property>
+ <Property name="webhcat.log.dir">.</Property>
+ <Property name="webhcat.log.file">webhcat.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Daily Rolling File Appender -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="standard" fileName="${sys:webhcat.log.dir}/${sys:webhcat.log.file}"
+ filePattern="${sys:webhcat.log.dir}/${sys:webhcat.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%-5p | %d{DATE} | %c | %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:webhcat.log.threshold}">
+ <AppenderRef ref="${sys:webhcat.root.logger}" level="${sys:webhcat.log.level}"/>
+ </Root>
+
+ <Logger name="com.sun.jersey" level="DEBUG">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ <Logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop" level="INFO">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hadoop.conf" level="WARN">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper" level="WARN">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ <Logger name="org.eclipse.jetty" level="INFO">
+ <AppenderRef ref="${sys:webhcat.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
----------------------------------------------------------------------
diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
index 73f470c..268c218 100644
--- a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
+++ b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
@@ -670,6 +670,8 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
if (!parseArguments(args)) {
return 1;
}
+ // specify the default log4j2 properties file.
+ System.setProperty("log4j.configurationFile", "hive-log4j2.xml");
conf = new Conf();
conf.init();
conn = new Conn(this);
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
new file mode 100644
index 0000000..93c16de
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
@@ -0,0 +1,136 @@
+package org.apache.hive.service.cli.operation;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.apache.hive.service.cli.CLIServiceClient;
+import org.apache.hive.service.cli.FetchOrientation;
+import org.apache.hive.service.cli.FetchType;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.SessionHandle;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests to verify operation logging layout for different modes.
+ */
+public class TestOperationLoggingLayout {
+ protected static HiveConf hiveConf;
+ protected static String tableName;
+ private File dataFile;
+ protected CLIServiceClient client;
+ protected static MiniHS2 miniHS2 = null;
+ protected static Map<String, String> confOverlay;
+ protected SessionHandle sessionHandle;
+ protected final String sql = "select * from " + tableName;
+ private final String sqlCntStar = "select count(*) from " + tableName;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ tableName = "TestOperationLoggingLayout_table";
+ hiveConf = new HiveConf();
+ hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "execution");
+ // We need to set the below parameter to test performance level logging
+ hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
+ miniHS2 = new MiniHS2(hiveConf);
+ confOverlay = new HashMap<String, String>();
+ confOverlay.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ miniHS2.start(confOverlay);
+ }
+
+ /**
+ * Open a session, and create a table for cases usage
+ *
+ * @throws Exception
+ */
+ @Before
+ public void setUp() throws Exception {
+ dataFile = new File(hiveConf.get("test.data.files"), "kv1.txt");
+ client = miniHS2.getServiceClient();
+ sessionHandle = setupSession();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // Cleanup
+ String queryString = "DROP TABLE " + tableName;
+ client.executeStatement(sessionHandle, queryString, null);
+
+ client.closeSession(sessionHandle);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ miniHS2.stop();
+ }
+
+ @Test
+ public void testSwitchLogLayout() throws Exception {
+ // verify whether the sql operation log is generated and fetch correctly.
+ OperationHandle operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
+ RowSet rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 1000,
+ FetchType.LOG);
+ Iterator<Object[]> iter = rowSetLog.iterator();
+ // non-verbose pattern is %-5p : %m%n. Look for " : "
+ while (iter.hasNext()) {
+ String row = iter.next()[0].toString();
+ Assert.assertEquals(true, row.matches("^(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
+ }
+
+ String queryString = "set hive.server2.logging.operation.level=verbose";
+ client.executeStatement(sessionHandle, queryString, null);
+ operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
+ // just check for first few lines, some log lines are multi-line strings which can break format
+ // checks below
+ rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10,
+ FetchType.LOG);
+ iter = rowSetLog.iterator();
+ // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"
+ while (iter.hasNext()) {
+ String row = iter.next()[0].toString();
+ // just check if the log line starts with date
+ Assert.assertEquals(true,
+ row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
+ }
+ }
+
+ private SessionHandle setupSession() throws Exception {
+ // Open a session
+ SessionHandle sessionHandle = client.openSession(null, null, null);
+
+ // Change lock manager to embedded mode
+ String queryString = "SET hive.lock.manager=" +
+ "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager";
+ client.executeStatement(sessionHandle, queryString, null);
+
+ // Drop the table if it exists
+ queryString = "DROP TABLE IF EXISTS " + tableName;
+ client.executeStatement(sessionHandle, queryString, null);
+
+ // Create a test table
+ queryString = "create table " + tableName + " (key int, value string)";
+ client.executeStatement(sessionHandle, queryString, null);
+
+ // Load data
+ queryString = "load data local inpath '" + dataFile + "' into table " + tableName;
+ client.executeStatement(sessionHandle, queryString, null);
+
+ // Precondition check: verify whether the table is created and data is fetched correctly.
+ OperationHandle operationHandle = client.executeStatement(sessionHandle, sql, null);
+ RowSet rowSetResult = client.fetchResults(operationHandle);
+ Assert.assertEquals(500, rowSetResult.numRows());
+ Assert.assertEquals(238, rowSetResult.iterator().next()[0]);
+ Assert.assertEquals("val_238", rowSetResult.iterator().next()[1]);
+
+ return sessionHandle;
+ }
+}
[12/50] [abbrv] hive git commit: HIVE-11563: Perflogger loglines are
repeated (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11563: Perflogger loglines are repeated (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0fab86c9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0fab86c9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0fab86c9
Branch: refs/heads/llap
Commit: 0fab86c9d7730275cb9962c24ce9d2f92f0c7150
Parents: d307abb
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri Aug 14 14:02:51 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri Aug 14 14:02:51 2015 -0700
----------------------------------------------------------------------
data/conf/hive-log4j2.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0fab86c9/data/conf/hive-log4j2.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
index a40be2c..ff8e4d3 100644
--- a/data/conf/hive-log4j2.xml
+++ b/data/conf/hive-log4j2.xml
@@ -95,7 +95,7 @@
<Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
<AppenderRef ref="${sys:hive.root.logger}"/>
</Logger>
- <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
+ <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}" additivity="false">
<AppenderRef ref="${sys:hive.root.logger}"/>
</Logger>
<Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
[29/50] [abbrv] hive git commit: HIVE-10697 :
ObjectInspectorConvertors#UnionConvertor does a faulty conversion (Swarnim
Kulkarni, reviewed by Hari Subramaniyan)
Posted by se...@apache.org.
HIVE-10697 : ObjectInspectorConvertors#UnionConvertor does a faulty conversion (Swarnim Kulkarni, reviewed by Hari Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2688b680
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2688b680
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2688b680
Branch: refs/heads/llap
Commit: 2688b6800f203ff5fd4b283e462594e9b4279975
Parents: c3c2bda
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Wed Aug 19 10:25:14 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Wed Aug 19 10:25:14 2015 -0700
----------------------------------------------------------------------
.../ObjectInspectorConverters.java | 31 +++----
.../SettableUnionObjectInspector.java | 4 +-
.../StandardUnionObjectInspector.java | 4 +-
.../TestObjectInspectorConverters.java | 89 +++++++++++++++++++-
4 files changed, 108 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
index 8ef8ce1..24b3d4e 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
@@ -424,8 +424,9 @@ public final class ObjectInspectorConverters {
UnionObjectInspector inputOI;
SettableUnionObjectInspector outputOI;
- List<? extends ObjectInspector> inputFields;
- List<? extends ObjectInspector> outputFields;
+ // Object inspectors for the tags for the input and output unionss
+ List<? extends ObjectInspector> inputTagsOIs;
+ List<? extends ObjectInspector> outputTagsOIs;
ArrayList<Converter> fieldConverters;
@@ -436,14 +437,14 @@ public final class ObjectInspectorConverters {
if (inputOI instanceof UnionObjectInspector) {
this.inputOI = (UnionObjectInspector)inputOI;
this.outputOI = outputOI;
- inputFields = this.inputOI.getObjectInspectors();
- outputFields = outputOI.getObjectInspectors();
+ inputTagsOIs = this.inputOI.getObjectInspectors();
+ outputTagsOIs = outputOI.getObjectInspectors();
// If the output has some extra fields, set them to NULL in convert().
- int minFields = Math.min(inputFields.size(), outputFields.size());
+ int minFields = Math.min(inputTagsOIs.size(), outputTagsOIs.size());
fieldConverters = new ArrayList<Converter>(minFields);
for (int f = 0; f < minFields; f++) {
- fieldConverters.add(getConverter(inputFields.get(f), outputFields.get(f)));
+ fieldConverters.add(getConverter(inputTagsOIs.get(f), outputTagsOIs.get(f)));
}
// Create an empty output object which will be populated when convert() is invoked.
@@ -461,18 +462,18 @@ public final class ObjectInspectorConverters {
return null;
}
- int minFields = Math.min(inputFields.size(), outputFields.size());
- // Convert the fields
- for (int f = 0; f < minFields; f++) {
- Object outputFieldValue = fieldConverters.get(f).convert(inputOI);
- outputOI.addField(output, (ObjectInspector)outputFieldValue);
- }
+ Object inputFieldValue = inputOI.getField(input);
+ Object inputFieldTag = inputOI.getTag(input);
+ Object outputFieldValue = null;
- // set the extra fields to null
- for (int f = minFields; f < outputFields.size(); f++) {
- outputOI.addField(output, null);
+ int inputFieldTagIndex = ((Byte)inputFieldTag).intValue();
+
+ if (inputFieldTagIndex >= 0 && inputFieldTagIndex < fieldConverters.size()) {
+ outputFieldValue = fieldConverters.get(inputFieldTagIndex).convert(inputFieldValue);
}
+ outputOI.addField(output, outputFieldValue);
+
return output;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
index a64aee0..564d8d6 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
@@ -29,6 +29,6 @@ public abstract class SettableUnionObjectInspector implements
/* Create an empty object */
public abstract Object create();
- /* Add fields to the object */
- public abstract Object addField(Object union, ObjectInspector oi);
+ /* Add field to the object */
+ public abstract Object addField(Object union, Object field);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
index d1b11e8..f26c9ec 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
@@ -124,9 +124,9 @@ public class StandardUnionObjectInspector extends SettableUnionObjectInspector {
}
@Override
- public Object addField(Object union, ObjectInspector oi) {
+ public Object addField(Object union, Object field) {
ArrayList<Object> a = (ArrayList<Object>) union;
- a.add(oi);
+ a.add(field);
return a;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
index 1185283..dd18517 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hive.serde2.objectinspector;
+import java.util.ArrayList;
+import java.util.List;
+
import junit.framework.TestCase;
import org.apache.hadoop.hive.common.type.HiveDecimal;
@@ -168,6 +171,90 @@ public class TestObjectInspectorConverters extends TestCase {
{(byte)'h', (byte)'i',(byte)'v',(byte)'e'}),
baConverter.convert(new Text("hive")));
assertEquals("BAConverter", null, baConverter.convert(null));
+
+ // Union
+ ArrayList<String> fieldNames = new ArrayList<String>();
+ fieldNames.add("firstInteger");
+ fieldNames.add("secondString");
+ fieldNames.add("thirdBoolean");
+ ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldObjectInspectors
+ .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
+
+ ArrayList<String> fieldNames2 = new ArrayList<String>();
+ fieldNames2.add("firstString");
+ fieldNames2.add("secondInteger");
+ fieldNames2.add("thirdBoolean");
+ ArrayList<ObjectInspector> fieldObjectInspectors2 = new ArrayList<ObjectInspector>();
+ fieldObjectInspectors2
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldObjectInspectors2
+ .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ fieldObjectInspectors2
+ .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
+
+ Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
+ ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
+
+ Object convertedObject0 = unionConverter0.convert(new StandardUnionObjectInspector.StandardUnion((byte)0, 1));
+ List<String> expectedObject0 = new ArrayList<String>();
+ expectedObject0.add("1");
+
+ assertEquals(expectedObject0, convertedObject0);
+
+ Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
+ ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
+
+ Object convertedObject1 = unionConverter1.convert(new StandardUnionObjectInspector.StandardUnion((byte)1, "1"));
+ List<Integer> expectedObject1 = new ArrayList<Integer>();
+ expectedObject1.add(1);
+
+ assertEquals(expectedObject1, convertedObject1);
+
+ Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
+ ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
+
+ Object convertedObject2 = unionConverter2.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true));
+ List<Boolean> expectedObject2 = new ArrayList<Boolean>();
+ expectedObject2.add(true);
+
+ assertEquals(expectedObject2, convertedObject2);
+
+ // Union (extra fields)
+ ArrayList<String> fieldNamesExtra = new ArrayList<String>();
+ fieldNamesExtra.add("firstInteger");
+ fieldNamesExtra.add("secondString");
+ fieldNamesExtra.add("thirdBoolean");
+ ArrayList<ObjectInspector> fieldObjectInspectorsExtra = new ArrayList<ObjectInspector>();
+ fieldObjectInspectorsExtra
+ .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+ fieldObjectInspectorsExtra
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldObjectInspectorsExtra
+ .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
+
+ ArrayList<String> fieldNamesExtra2 = new ArrayList<String>();
+ fieldNamesExtra2.add("firstString");
+ fieldNamesExtra2.add("secondInteger");
+ ArrayList<ObjectInspector> fieldObjectInspectorsExtra2 = new ArrayList<ObjectInspector>();
+ fieldObjectInspectorsExtra2
+ .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ fieldObjectInspectorsExtra2
+ .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
+
+ Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra),
+ ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2));
+
+ Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true));
+ List<Object> expectedObjectExtra = new ArrayList<Object>();
+ expectedObjectExtra.add(null);
+
+ assertEquals(expectedObjectExtra, convertedObjectExtra); // we should get back null
+
} catch (Throwable e) {
e.printStackTrace();
throw e;
@@ -192,4 +279,4 @@ public class TestObjectInspectorConverters extends TestCase {
VarcharTypeInfo vcParams = (VarcharTypeInfo) poi.getTypeInfo();
assertEquals("varchar length doesn't match", 5, vcParams.getLength());
}
-}
+}
\ No newline at end of file
[02/50] [abbrv] hive git commit: HIVE-11541: ORC: Split Strategy
should depend on global file count,
not per-partition (Gopal V reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-11541: ORC: Split Strategy should depend on global file count, not per-partition (Gopal V reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f26b2569
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f26b2569
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f26b2569
Branch: refs/heads/llap
Commit: f26b2569198fbeceaf17a5a77c59eccf5175935c
Parents: db46e6e
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Thu Aug 13 12:35:29 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Thu Aug 13 12:35:29 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f26b2569/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 4e6dd7a..fe2eccd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -483,7 +483,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
private FileInfo verifyCachedFileInfo(FileStatus file) {
- context.numFilesCounter.incrementAndGet();
FileInfo fileInfo = Context.footerCache.getIfPresent(file.getPath());
if (fileInfo != null) {
if (isDebugEnabled) {
@@ -671,6 +670,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
int numFiles = children.size();
long avgFileSize = totalFileSize / numFiles;
+ int totalFiles = context.numFilesCounter.addAndGet(numFiles);
switch(context.splitStrategyKind) {
case BI:
// BI strategy requested through config
@@ -684,7 +684,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
break;
default:
// HYBRID strategy
- if (avgFileSize > context.maxSize || numFiles <= context.minSplits) {
+ if (avgFileSize > context.maxSize || totalFiles <= context.minSplits) {
splitStrategy = new ETLSplitStrategy(context, fs, dir, children, isOriginal, deltas,
covered);
} else {
[46/50] [abbrv] hive git commit: HIVE-11521 : Loop optimization for
SIMD in logical operators (Teddy Choi via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11521 : Loop optimization for SIMD in logical operators (Teddy Choi via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7796dd67
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7796dd67
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7796dd67
Branch: refs/heads/llap
Commit: 7796dd671a0c28cbcf2f1fc51b01fcd99ed49561
Parents: e2a1764
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Tue Aug 11 07:05:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 24 10:24:39 2015 -0700
----------------------------------------------------------------------
.../vectorization/VectorizationBench.java | 93 ++++++++++++++++++++
.../ql/exec/vector/expressions/ColAndCol.java | 34 +++----
.../ql/exec/vector/expressions/ColOrCol.java | 42 ++++-----
.../hive/ql/exec/vector/expressions/NotCol.java | 14 +--
4 files changed, 140 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7796dd67/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java
----------------------------------------------------------------------
diff --git a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java
index 391d052..0e880c6 100644
--- a/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java
+++ b/itests/hive-jmh/src/main/java/org/apache/hive/benchmark/vectorization/VectorizationBench.java
@@ -17,7 +17,10 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.ColAndCol;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.ColOrCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColDivideLongColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.NotCol;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DoubleColAddDoubleColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DoubleColDivideDoubleColumn;
@@ -124,6 +127,34 @@ public class VectorizationBench {
return columnVector;
}
+ protected LongColumnVector getBooleanLongColumnVector() {
+ LongColumnVector columnVector = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ Random random = new Random();
+ for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ columnVector.vector[i] = random.nextInt(2);
+ }
+ return columnVector;
+ }
+
+ protected LongColumnVector getBooleanRepeatingLongColumnVector() {
+ LongColumnVector columnVector = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ columnVector.fill(1);
+ return columnVector;
+ }
+
+ protected LongColumnVector getBooleanLongColumnVectorWithNull() {
+ LongColumnVector columnVector = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
+ columnVector.noNulls = false;
+ Random random = new Random();
+ for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) {
+ if (i % 100 == 0) {
+ columnVector.isNull[i] = true;
+ }
+ columnVector.vector[i] = random.nextInt(2);
+ }
+ return columnVector;
+ }
+
protected DoubleColumnVector getDoubleColumnVector() {
DoubleColumnVector columnVector = new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
Random random = new Random();
@@ -209,6 +240,68 @@ public class VectorizationBench {
}
}
+ public static class ColAndColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
+ getBooleanLongColumnVector());
+ expression = new ColAndCol(0, 1, 2);
+ }
+ }
+
+ public static class ColAndRepeatingColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
+ getBooleanRepeatingLongColumnVector());
+ expression = new ColAndCol(0, 1, 2);
+ }
+ }
+
+ public static class RepeatingColAndColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanRepeatingLongColumnVector(),
+ getBooleanLongColumnVector());
+ expression = new ColAndCol(0, 1, 2);
+ }
+ }
+
+ public static class ColOrColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
+ getBooleanLongColumnVector());
+ expression = new ColOrCol(0, 1, 2);
+ }
+ }
+
+ public static class ColOrRepeatingColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanLongColumnVector(),
+ getBooleanRepeatingLongColumnVector());
+ expression = new ColOrCol(0, 1, 2);
+ }
+ }
+
+ public static class RepeatingColOrColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 2, getBooleanRepeatingLongColumnVector(),
+ getBooleanLongColumnVector());
+ expression = new ColOrCol(0, 1, 2);
+ }
+ }
+
+ public static class NotColBench extends AbstractExpression {
+ @Override
+ public void setup() {
+ rowBatch = buildRowBatch(new LongColumnVector(), 1, getBooleanLongColumnVector());
+ expression = new NotCol(0, 1);
+ }
+ }
+
public static void main(String[] args) throws RunnerException {
Options opt = new OptionsBuilder().include(".*" + VectorizationBench.class.getSimpleName() +
".*").build();
http://git-wip-us.apache.org/repos/asf/hive/blob/7796dd67/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColAndCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColAndCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColAndCol.java
index 72df220..ff7371d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColAndCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColAndCol.java
@@ -64,6 +64,8 @@ public class ColAndCol extends VectorExpression {
return;
}
+ long vector1Value = vector1[0];
+ long vector2Value = vector2[0];
if (inputColVector1.noNulls && inputColVector2.noNulls) {
if ((inputColVector1.isRepeating) && (inputColVector2.isRepeating)) {
// All must be selected otherwise size would be zero
@@ -74,11 +76,11 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
}
}
outV.isRepeating = false;
@@ -86,11 +88,11 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
}
}
outV.isRepeating = false;
@@ -120,12 +122,12 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = (vector1[0] == 1) && inputColVector2.isNull[i];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = (vector1[0] == 1) && inputColVector2.isNull[i];
}
}
@@ -134,12 +136,12 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = (vector1[i] == 1) && inputColVector2.isNull[0];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = (vector1[i] == 1) && inputColVector2.isNull[0];
}
}
@@ -172,12 +174,12 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = inputColVector1.isNull[0] && (vector2[i] == 1);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = inputColVector1.isNull[0] && (vector2[i] == 1);
}
}
@@ -186,12 +188,12 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = inputColVector1.isNull[i] && (vector2[0] == 1);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = inputColVector1.isNull[i] && (vector2[0] == 1);
}
}
@@ -226,14 +228,14 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = ((vector1[0] == 1) && inputColVector2.isNull[i])
|| (inputColVector1.isNull[0] && (vector2[i] == 1))
|| (inputColVector1.isNull[0] && inputColVector2.isNull[i]);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] & vector2[i];
+ outputVector[i] = vector1Value & vector2[i];
outV.isNull[i] = ((vector1[0] == 1) && inputColVector2.isNull[i])
|| (inputColVector1.isNull[0] && (vector2[i] == 1))
|| (inputColVector1.isNull[0] && inputColVector2.isNull[i]);
@@ -244,14 +246,14 @@ public class ColAndCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = ((vector1[i] == 1) && inputColVector2.isNull[0])
|| (inputColVector1.isNull[i] && (vector2[0] == 1))
|| (inputColVector1.isNull[i] && inputColVector2.isNull[0]);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] & vector2[0];
+ outputVector[i] = vector1[i] & vector2Value;
outV.isNull[i] = ((vector1[i] == 1) && inputColVector2.isNull[0])
|| (inputColVector1.isNull[i] && (vector2[0] == 1))
|| (inputColVector1.isNull[i] && inputColVector2.isNull[0]);
http://git-wip-us.apache.org/repos/asf/hive/blob/7796dd67/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColOrCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColOrCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColOrCol.java
index 5e23446..60ed2d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColOrCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ColOrCol.java
@@ -67,6 +67,8 @@ public class ColOrCol extends VectorExpression {
return;
}
+ long vector1Value = vector1[0];
+ long vector2Value = vector2[0];
if (inputColVector1.noNulls && inputColVector2.noNulls) {
if ((inputColVector1.isRepeating) && (inputColVector2.isRepeating)) {
// All must be selected otherwise size would be zero
@@ -77,11 +79,11 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
}
}
outV.isRepeating = false;
@@ -89,11 +91,11 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
}
}
outV.isRepeating = false;
@@ -123,13 +125,13 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] | vector2[i];
- outV.isNull[i] = (vector1[0] == 0) && inputColVector2.isNull[i];
+ outputVector[i] = vector1Value | vector2[i];
+ outV.isNull[i] = (vector1Value == 0) && inputColVector2.isNull[i];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] | vector2[i];
- outV.isNull[i] = (vector1[0] == 0) && inputColVector2.isNull[i];
+ outputVector[i] = vector1Value | vector2[i];
+ outV.isNull[i] = (vector1Value == 0) && inputColVector2.isNull[i];
}
}
outV.isRepeating = false;
@@ -137,12 +139,12 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
outV.isNull[i] = (vector1[i] == 0) && inputColVector2.isNull[0];
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
outV.isNull[i] = (vector1[i] == 0) && inputColVector2.isNull[0];
}
}
@@ -175,12 +177,12 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
outV.isNull[i] = inputColVector1.isNull[0] && (vector2[i] == 0);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
outV.isNull[i] = inputColVector1.isNull[0] && (vector2[i] == 0);
}
}
@@ -189,13 +191,13 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] | vector2[0];
- outV.isNull[i] = inputColVector1.isNull[i] && (vector2[0] == 0);
+ outputVector[i] = vector1[i] | vector2Value;
+ outV.isNull[i] = inputColVector1.isNull[i] && (vector2Value == 0);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] | vector2[0];
- outV.isNull[i] = inputColVector1.isNull[i] && (vector2[0] == 0);
+ outputVector[i] = vector1[i] | vector2Value;
+ outV.isNull[i] = inputColVector1.isNull[i] && (vector2Value == 0);
}
}
outV.isRepeating = false;
@@ -229,14 +231,14 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
outV.isNull[i] = ((vector1[0] == 0) && inputColVector2.isNull[i])
|| (inputColVector1.isNull[0] && (vector2[i] == 0))
|| (inputColVector1.isNull[0] && inputColVector2.isNull[i]);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] | vector2[i];
+ outputVector[i] = vector1Value | vector2[i];
outV.isNull[i] = ((vector1[0] == 0) && inputColVector2.isNull[i])
|| (inputColVector1.isNull[0] && (vector2[i] == 0))
|| (inputColVector1.isNull[0] && inputColVector2.isNull[i]);
@@ -247,14 +249,14 @@ public class ColOrCol extends VectorExpression {
if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
outV.isNull[i] = ((vector1[i] == 0) && inputColVector2.isNull[0])
|| (inputColVector1.isNull[i] && (vector2[0] == 0))
|| (inputColVector1.isNull[i] && inputColVector2.isNull[0]);
}
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] | vector2[0];
+ outputVector[i] = vector1[i] | vector2Value;
outV.isNull[i] = ((vector1[i] == 0) && inputColVector2.isNull[0])
|| (inputColVector1.isNull[i] && (vector2[0] == 0))
|| (inputColVector1.isNull[i] && inputColVector2.isNull[0]);
http://git-wip-us.apache.org/repos/asf/hive/blob/7796dd67/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NotCol.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NotCol.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NotCol.java
index 604d154..ea2a434 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NotCol.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NotCol.java
@@ -62,17 +62,17 @@ public class NotCol extends VectorExpression {
outV.noNulls = true;
if (inputColVector.isRepeating) {
outV.isRepeating = true;
- // mask out all but low order bit with "& 1" so NOT 1 yields 0, NOT 0 yields 1
- outputVector[0] = ~vector[0] & 1;
+ // 0 XOR 1 yields 1, 1 XOR 1 yields 0
+ outputVector[0] = vector[0] ^ 1;
} else if (batch.selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = ~vector[i] & 1;
+ outputVector[i] = vector[i] ^ 1;
}
outV.isRepeating = false;
} else {
for (int i = 0; i != n; i++) {
- outputVector[i] = ~vector[i] & 1;
+ outputVector[i] = vector[i] ^ 1;
}
outV.isRepeating = false;
}
@@ -80,19 +80,19 @@ public class NotCol extends VectorExpression {
outV.noNulls = false;
if (inputColVector.isRepeating) {
outV.isRepeating = true;
- outputVector[0] = ~vector[0] & 1;
+ outputVector[0] = vector[0] ^ 1;
outV.isNull[0] = inputColVector.isNull[0];
} else if (batch.selectedInUse) {
outV.isRepeating = false;
for (int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = ~vector[i] & 1;
+ outputVector[i] = vector[i] ^ 1;
outV.isNull[i] = inputColVector.isNull[i];
}
} else {
outV.isRepeating = false;
for (int i = 0; i != n; i++) {
- outputVector[i] = ~vector[i] & 1;
+ outputVector[i] = vector[i] ^ 1;
outV.isNull[i] = inputColVector.isNull[i];
}
}
[38/50] [abbrv] hive git commit: HIVE-5277: HBase handler skips rows
with null valued first cells when only row key is selected (Swarnim Kulkarni,
reviewed by Xuefu Zhang)
Posted by se...@apache.org.
HIVE-5277: HBase handler skips rows with null valued first cells when only row key is selected (Swarnim Kulkarni, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1030ae71
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1030ae71
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1030ae71
Branch: refs/heads/llap
Commit: 1030ae719e2bbea47d8f2835244589621ab126fb
Parents: b4a6ac9
Author: Swarnim Kulkarni <ku...@gmail.com>
Authored: Fri Aug 21 11:21:08 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 21 11:21:08 2015 -0700
----------------------------------------------------------------------
.../hive/hbase/HiveHBaseInputFormatUtil.java | 50 +++++----
.../queries/positive/hbase_null_first_col.q | 22 ++++
.../results/positive/hbase_null_first_col.q.out | 109 +++++++++++++++++++
3 files changed, 158 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
index 0524572..c002070 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
@@ -18,9 +18,18 @@
package org.apache.hadoop.hive.hbase;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
@@ -28,12 +37,6 @@ import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.mapred.JobConf;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* Util code common between HiveHBaseTableInputFormat and HiveHBaseTableSnapshotInputFormat.
*/
@@ -95,26 +98,27 @@ class HiveHBaseInputFormatUtil {
}
}
- // The HBase table's row key maps to a Hive table column. In the corner case when only the
- // row key column is selected in Hive, the HBase Scan will be empty i.e. no column family/
- // column qualifier will have been added to the scan. We arbitrarily add at least one column
- // to the HBase scan so that we can retrieve all of the row keys and return them as the Hive
- // tables column projection.
+ // If we have cases where we are running a query like count(key) or count(*),
+ // in such cases, the readColIDs is either empty(for count(*)) or has just the
+ // key column in it. In either case, nothing gets added to the scan. So if readAllColumns is
+ // true, we are going to add all columns. Else we are just going to add a key filter to run a
+ // count only on the keys
if (empty) {
- for (ColumnMapping colMap: columnMappings) {
- if (colMap.hbaseRowKey || colMap.hbaseTimestamp) {
- continue;
- }
-
- if (colMap.qualifierName == null) {
- scan.addFamily(colMap.familyNameBytes);
- } else {
- scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
- }
+ if (readAllColumns) {
+ for (ColumnMapping colMap: columnMappings) {
+ if (colMap.hbaseRowKey || colMap.hbaseTimestamp) {
+ continue;
+ }
- if (!readAllColumns) {
- break;
+ if (colMap.qualifierName == null) {
+ scan.addFamily(colMap.familyNameBytes);
+ } else {
+ scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
+ }
}
+ } else {
+ // Add a filter to just do a scan on the keys so that we pick up everything
+ scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new KeyOnlyFilter()));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_null_first_col.q b/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
new file mode 100644
index 0000000..0d9ff56
--- /dev/null
+++ b/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
@@ -0,0 +1,22 @@
+DROP TABLE src_null;
+DROP TABLE hbase_null;
+
+CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null;
+
+CREATE TABLE hbase_null(key string, col1 string, col2 string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
+);
+
+SELECT d, a, c FROM src_null;
+
+INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null;
+
+SELECT COUNT(d) FROM src_null;
+SELECT COUNT(key) FROM hbase_null;
+SELECT COUNT(*) FROM hbase_null;
+
+DROP TABLE src_null;
+DROP TABLE hbase_null;
http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out b/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
new file mode 100644
index 0000000..bb4491b
--- /dev/null
+++ b/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
@@ -0,0 +1,109 @@
+PREHOOK: query: DROP TABLE src_null
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE src_null
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE hbase_null
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_null
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_null
+POSTHOOK: query: CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_null
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_null
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_null
+PREHOOK: query: CREATE TABLE hbase_null(key string, col1 string, col2 string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_null
+POSTHOOK: query: CREATE TABLE hbase_null(key string, col1 string, col2 string)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_null
+PREHOOK: query: SELECT d, a, c FROM src_null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_null
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT d, a, c FROM src_null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_null
+#### A masked pattern was here ####
+0 1.0 same
+1 1.0 same
+2 1.0 same
+3 1.0 same
+4 1.0 same
+5 NULL same
+6 NULL same
+7 1.0 same
+8 1.0 same
+9 1.0 same
+PREHOOK: query: INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_null
+PREHOOK: Output: default@hbase_null
+POSTHOOK: query: INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_null
+POSTHOOK: Output: default@hbase_null
+PREHOOK: query: SELECT COUNT(d) FROM src_null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_null
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(d) FROM src_null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_null
+#### A masked pattern was here ####
+10
+PREHOOK: query: SELECT COUNT(key) FROM hbase_null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_null
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(key) FROM hbase_null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_null
+#### A masked pattern was here ####
+10
+PREHOOK: query: SELECT COUNT(*) FROM hbase_null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_null
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT COUNT(*) FROM hbase_null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_null
+#### A masked pattern was here ####
+10
+PREHOOK: query: DROP TABLE src_null
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_null
+PREHOOK: Output: default@src_null
+POSTHOOK: query: DROP TABLE src_null
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_null
+POSTHOOK: Output: default@src_null
+PREHOOK: query: DROP TABLE hbase_null
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_null
+PREHOOK: Output: default@hbase_null
+POSTHOOK: query: DROP TABLE hbase_null
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_null
+POSTHOOK: Output: default@hbase_null
[50/50] [abbrv] hive git commit: HIVE-11643 : LLAP: merge master into
branch (Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-11643 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/88248fb6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/88248fb6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/88248fb6
Branch: refs/heads/llap
Commit: 88248fb6b41833cea2fce89cc14244605cf943ee
Parents: b97a076 284859c
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Aug 25 14:27:56 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Aug 25 14:27:56 2015 -0700
----------------------------------------------------------------------
accumulo-handler/pom.xml | 4 -
.../apache/hadoop/hive/ant/GenVectorCode.java | 2 +
.../java/org/apache/hive/beeline/BeeLine.java | 13 +-
.../org/apache/hive/beeline/BeeLineOpts.java | 12 +-
.../org/apache/hive/beeline/HiveSchemaTool.java | 14 +-
.../src/main/resources/beeline-log4j.properties | 24 -
beeline/src/main/resources/beeline-log4j2.xml | 40 +
bin/ext/beeline.sh | 2 +-
bin/hive | 3 +
common/pom.xml | 22 +-
.../apache/hadoop/hive/common/JavaUtils.java | 11 +-
.../org/apache/hadoop/hive/common/LogUtils.java | 18 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 92 +-
common/src/main/resources/hive-log4j2.xml | 111 +
.../hadoop/hive/conf/TestHiveLogging.java | 8 +-
.../resources/hive-exec-log4j-test.properties | 59 -
.../test/resources/hive-exec-log4j2-test.xml | 86 +
.../test/resources/hive-log4j-test.properties | 71 -
common/src/test/resources/hive-log4j2-test.xml | 95 +
data/conf/hive-log4j-old.properties | 82 -
data/conf/hive-log4j.properties | 97 -
data/conf/hive-log4j2.xml | 147 +
data/conf/hive-site.xml | 6 -
data/conf/spark/log4j.properties | 24 -
data/conf/spark/log4j2.xml | 74 +
docs/xdocs/language_manual/cli.xml | 2 +-
.../hive/hbase/HiveHBaseInputFormatUtil.java | 50 +-
.../queries/positive/hbase_null_first_col.q | 22 +
.../results/positive/hbase_null_first_col.q.out | 109 +
.../test/results/positive/hbase_timestamp.q.out | 8 +-
hcatalog/bin/hcat_server.sh | 2 +-
hcatalog/bin/templeton.cmd | 4 +-
hcatalog/scripts/hcat_server_start.sh | 2 +-
.../content/xdocs/configuration.xml | 2 +-
.../src/documentation/content/xdocs/install.xml | 2 +-
.../deployers/config/hive/hive-log4j.properties | 88 -
.../deployers/config/hive/hive-log4j2.xml | 111 +
.../deployers/config/hive/hive-site.mysql.xml | 22 +
.../templeton/deployers/start_hive_services.sh | 2 +-
.../hive/hcatalog/streaming/TestStreaming.java | 54 +-
.../webhcat/svr/src/main/bin/webhcat_server.sh | 4 +-
.../src/main/config/webhcat-log4j.properties | 45 -
.../svr/src/main/config/webhcat-log4j2.xml | 75 +
.../main/java/org/apache/hive/hplsql/Exec.java | 2 +
.../vectorization/VectorizationBench.java | 93 +
.../hive/metastore/TestHiveMetaStore.java | 66 +
.../TestOperationLoggingAPIWithMr.java | 2 -
.../TestOperationLoggingAPIWithTez.java | 2 -
.../operation/TestOperationLoggingLayout.java | 134 +
itests/pom.xml | 2 +-
itests/qtest/pom.xml | 26 +-
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/QTestUtil.java | 63 +-
jdbc/pom.xml | 1 +
.../org/apache/hive/jdbc/HiveConnection.java | 30 +-
.../org/apache/hive/jdbc/HiveStatement.java | 2 +-
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 117 +-
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 104 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 26 +-
.../hive/metastore/HouseKeeperService.java | 39 +
.../hadoop/hive/metastore/MetaStoreUtils.java | 17 +-
.../hadoop/hive/metastore/ObjectStore.java | 7 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 178 +-
.../metastore/txn/TestCompactionTxnHandler.java | 40 +-
.../hive/metastore/txn/TestTxnHandler.java | 73 +-
packaging/src/main/assembly/bin.xml | 17 +-
pom.xml | 39 +-
ql/pom.xml | 17 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 28 +-
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 11 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 64 +-
.../hadoop/hive/ql/exec/KeyWrapperFactory.java | 18 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 25 -
.../hadoop/hive/ql/exec/ScriptOperator.java | 85 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 26 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 29 +-
.../hive/ql/exec/mr/HadoopJobExecHelper.java | 20 +-
.../persistence/HybridHashTableContainer.java | 6 +
.../spark/status/impl/LocalSparkJobStatus.java | 2 +-
.../spark/status/impl/RemoteSparkJobStatus.java | 2 +-
.../ql/exec/vector/VectorizationContext.java | 56 +-
.../BRoundWithNumDigitsDoubleToDouble.java | 42 +
.../expressions/CastStringGroupToString.java | 40 +
.../ql/exec/vector/expressions/ColAndCol.java | 34 +-
.../ql/exec/vector/expressions/ColOrCol.java | 42 +-
.../ql/exec/vector/expressions/DecimalUtil.java | 18 +
.../vector/expressions/FilterExprAndExpr.java | 8 +-
.../vector/expressions/FilterExprOrExpr.java | 140 +-
...FuncBRoundWithNumDigitsDecimalToDecimal.java | 40 +
.../FuncRoundWithNumDigitsDecimalToDecimal.java | 14 +-
.../ql/exec/vector/expressions/MathExpr.java | 22 +
.../hive/ql/exec/vector/expressions/NotCol.java | 14 +-
.../ql/exec/vector/expressions/StringExpr.java | 49 +-
.../hadoop/hive/ql/hooks/LineageInfo.java | 9 +-
.../hadoop/hive/ql/hooks/LineageLogger.java | 44 +-
.../hive/ql/hooks/PostExecOrcFileDump.java | 120 +
.../ql/hooks/PostExecTezSummaryPrinter.java | 72 +
.../hive/ql/io/CombineHiveInputFormat.java | 39 -
.../apache/hadoop/hive/ql/io/orc/FileDump.java | 4 +
.../apache/hadoop/hive/ql/io/orc/OrcConf.java | 2 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 82 +-
.../apache/hadoop/hive/ql/io/orc/OrcSerde.java | 6 +-
.../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 7 +-
.../apache/hadoop/hive/ql/io/orc/Reader.java | 7 +
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 39 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 18 +-
.../parquet/read/DataWritableReadSupport.java | 10 +-
.../ql/io/rcfile/stats/PartialScanTask.java | 20 +-
.../hive/ql/io/sarg/ConvertAstToSearchArg.java | 14 +-
.../hadoop/hive/ql/lib/DefaultGraphWalker.java | 2 +-
.../hadoop/hive/ql/lib/PreOrderOnceWalker.java | 44 +
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 12 +-
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 37 +-
.../hadoop/hive/ql/log/HiveEventCounter.java | 135 +
.../apache/hadoop/hive/ql/log/NullAppender.java | 63 +
.../ql/log/PidDailyRollingFileAppender.java | 33 -
.../hive/ql/log/PidFilePatternConverter.java | 62 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 77 +-
.../hadoop/hive/ql/metadata/Partition.java | 2 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
.../optimizer/ConstantPropagateProcFactory.java | 217 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 2 -
.../hive/ql/optimizer/GroupByOptimizer.java | 58 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 6 +
.../hive/ql/optimizer/PointLookupOptimizer.java | 280 +
.../calcite/reloperators/HiveFilter.java | 2 +-
.../calcite/translator/ExprNodeConverter.java | 44 +-
.../calcite/translator/TypeConverter.java | 13 +-
.../correlation/AbstractCorrelationProcCtx.java | 7 +
.../correlation/CorrelationUtilities.java | 11 +-
.../correlation/ReduceSinkDeDuplication.java | 6 +-
.../ql/optimizer/index/RewriteCanApplyCtx.java | 8 +-
.../ql/optimizer/lineage/ExprProcFactory.java | 9 +-
.../hive/ql/optimizer/lineage/LineageCtx.java | 34 +-
.../ql/optimizer/lineage/OpProcFactory.java | 10 +-
.../ql/optimizer/pcr/PcrExprProcFactory.java | 154 +-
.../physical/NullScanTaskDispatcher.java | 6 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 17 +-
.../hive/ql/optimizer/ppr/OpProcFactory.java | 3 +-
.../hive/ql/optimizer/ppr/PartitionPruner.java | 69 +-
.../apache/hadoop/hive/ql/parse/ASTNode.java | 18 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 40 +-
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 6 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 11 +
.../hadoop/hive/ql/parse/IdentifiersParser.g | 16 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 45 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 42 +-
.../hive/ql/plan/ExprNodeConstantDesc.java | 29 +-
.../apache/hadoop/hive/ql/plan/FilterDesc.java | 14 +-
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 -
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 9 +-
.../hive/ql/ppd/SyntheticJoinPredicate.java | 4 +-
.../hadoop/hive/ql/processors/SetProcessor.java | 4 +
.../hadoop/hive/ql/session/SessionState.java | 10 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 44 +-
.../hive/ql/txn/AcidHouseKeeperService.java | 104 +
.../hive/ql/txn/compactor/CompactorMR.java | 6 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 1 +
.../hadoop/hive/ql/txn/compactor/Worker.java | 2 +-
.../hive/ql/udf/generic/GenericUDAFStd.java | 2 +
.../ql/udf/generic/GenericUDAFVariance.java | 2 +
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 14 +-
.../hive/ql/udf/generic/GenericUDFBRound.java | 68 +
.../ql/udf/generic/GenericUDFBaseNumeric.java | 4 +-
.../hive/ql/udf/generic/GenericUDFBasePad.java | 8 +-
.../hive/ql/udf/generic/GenericUDFIn.java | 14 +-
.../hive/ql/udf/generic/GenericUDFNvl.java | 2 +-
.../hive/ql/udf/generic/GenericUDFOPAnd.java | 63 +-
.../hive/ql/udf/generic/GenericUDFOPEqual.java | 4 +
.../generic/GenericUDFOPEqualOrGreaterThan.java | 4 +
.../generic/GenericUDFOPEqualOrLessThan.java | 4 +
.../ql/udf/generic/GenericUDFOPGreaterThan.java | 4 +
.../ql/udf/generic/GenericUDFOPLessThan.java | 4 +
.../ql/udf/generic/GenericUDFOPNotEqual.java | 5 +
.../ql/udf/generic/GenericUDFOPNotNull.java | 4 +
.../hive/ql/udf/generic/GenericUDFOPNull.java | 4 +
.../hive/ql/udf/generic/GenericUDFOPOr.java | 63 +-
.../hive/ql/udf/generic/GenericUDFRound.java | 41 +-
.../hive/ql/udf/generic/GenericUDFStruct.java | 25 +-
.../hadoop/hive/ql/udf/generic/RoundUtils.java | 14 +
ql/src/main/resources/hive-exec-log4j2.xml | 110 +
.../apache/hadoop/hive/ql/TestTxnCommands.java | 21 +
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 1 +
.../hadoop/hive/ql/exec/TestOperators.java | 16 +
.../exec/vector/TestVectorizationContext.java | 93 +
.../TestVectorLogicalExpressions.java | 282 +
.../hive/ql/io/orc/TestInputOutputFormat.java | 95 +-
.../hive/ql/lockmgr/TestDbTxnManager.java | 74 +-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 2 +-
.../hadoop/hive/ql/log/TestLog4j2Appenders.java | 95 +
.../hadoop/hive/ql/metadata/StringAppender.java | 128 +
.../hadoop/hive/ql/metadata/TestHive.java | 50 +-
.../ql/udf/generic/TestGenericUDFBRound.java | 202 +
.../clientnegative/alter_table_wrong_location.q | 4 +
.../queries/clientnegative/ctas_noemptyfolder.q | 10 +
.../clientnegative/mismatch_columns_insertion.q | 4 +
.../queries/clientnegative/nvl_mismatch_type.q | 20 +
.../annotate_stats_deep_filters.q | 3 +-
.../clientpositive/authorization_1_sql_std.q | 4 +
.../clientpositive/cast_tinyint_to_double.q | 7 +
ql/src/test/queries/clientpositive/cbo_rp_gby.q | 24 +
.../queries/clientpositive/cbo_rp_gby_empty.q | 30 +
.../test/queries/clientpositive/cbo_rp_insert.q | 17 +
.../test/queries/clientpositive/cbo_rp_join.q | 65 +
.../test/queries/clientpositive/cbo_rp_limit.q | 16 +
.../queries/clientpositive/cbo_rp_semijoin.q | 17 +
.../clientpositive/cbo_rp_simple_select.q | 56 +
.../test/queries/clientpositive/cbo_rp_stats.q | 10 +
.../queries/clientpositive/cbo_rp_subq_exists.q | 67 +
.../queries/clientpositive/cbo_rp_subq_in.q | 56 +
.../queries/clientpositive/cbo_rp_subq_not_in.q | 81 +
.../queries/clientpositive/cbo_rp_udf_udaf.q | 20 +
.../test/queries/clientpositive/cbo_rp_union.q | 14 +
.../test/queries/clientpositive/cbo_rp_views.q | 46 +
.../queries/clientpositive/cbo_rp_windowing.q | 21 +
.../clientpositive/columnstats_quoting.q | 8 +
.../test/queries/clientpositive/create_like.q | 12 +
.../queries/clientpositive/flatten_and_or.q | 17 +
.../queries/clientpositive/folder_predicate.q | 32 +
.../clientpositive/insertoverwrite_bucket.q | 28 +
ql/src/test/queries/clientpositive/lineage3.q | 15 +
.../queries/clientpositive/macro_duplicate.q | 10 +
.../test/queries/clientpositive/orc_file_dump.q | 57 +
.../test/queries/clientpositive/orc_ppd_basic.q | 177 +
.../queries/clientpositive/selectDistinctStar.q | 2 +
ql/src/test/queries/clientpositive/structin.q | 23 +
ql/src/test/queries/clientpositive/udf_bround.q | 44 +
.../clientpositive/unionall_unbalancedppd.q | 3 +
.../test/queries/clientpositive/vector_bround.q | 14 +
.../clientpositive/vector_cast_constant.q | 4 +-
.../queries/clientpositive/vectorized_casts.q | 6 +
.../alter_table_wrong_location.q.out | 9 +
.../clientnegative/char_pad_convert_fail0.q.out | 2 +-
.../clientnegative/char_pad_convert_fail1.q.out | 2 +-
.../clientnegative/char_pad_convert_fail3.q.out | 2 +-
.../clientnegative/ctas_noemptyfolder.q.out | 19 +
.../mismatch_columns_insertion.q.out | 9 +
.../clientnegative/nvl_mismatch_type.q.out | 43 +
.../alter_partition_coltype.q.out | 12 +-
.../annotate_stats_deep_filters.q.out | 4 +-
.../clientpositive/annotate_stats_filter.q.out | 18 +-
.../authorization_1_sql_std.q.out | 11 +
.../clientpositive/cast_tinyint_to_double.q.out | 38 +
.../results/clientpositive/cbo_rp_gby.q.out | 124 +
.../clientpositive/cbo_rp_gby_empty.q.out | 77 +
.../results/clientpositive/cbo_rp_insert.q.out | 89 +
.../results/clientpositive/cbo_rp_join.q.out | 15028 +++++++++++++++++
.../results/clientpositive/cbo_rp_limit.q.out | 90 +
.../clientpositive/cbo_rp_semijoin.q.out | 440 +
.../clientpositive/cbo_rp_simple_select.q.out | 755 +
.../results/clientpositive/cbo_rp_stats.q.out | 14 +
.../clientpositive/cbo_rp_subq_exists.q.out | 297 +
.../results/clientpositive/cbo_rp_subq_in.q.out | 151 +
.../clientpositive/cbo_rp_subq_not_in.q.out | 365 +
.../clientpositive/cbo_rp_udf_udaf.q.out | 125 +
.../results/clientpositive/cbo_rp_union.q.out | 920 +
.../results/clientpositive/cbo_rp_views.q.out | 237 +
.../clientpositive/cbo_rp_windowing.q.out | 293 +
.../clientpositive/columnstats_quoting.q.out | 114 +
.../results/clientpositive/create_like.q.out | 66 +
.../results/clientpositive/decimal_udf.q.out | 18 +-
.../clientpositive/dynamic_rdd_cache.q.out | 6 +-
.../clientpositive/filter_cond_pushdown.q.out | 32 +-
.../clientpositive/filter_join_breaktask.q.out | 12 +-
.../results/clientpositive/flatten_and_or.q.out | 66 +
.../test/results/clientpositive/fold_when.q.out | 16 +-
.../clientpositive/folder_predicate.q.out | 368 +
.../groupby_multi_single_reducer2.q.out | 2 +-
.../groupby_multi_single_reducer3.q.out | 12 +-
.../clientpositive/groupby_sort_1_23.q.out | 56 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 56 +-
.../clientpositive/infer_const_type.q.out | 7 +-
.../clientpositive/input_testxpath2.q.out | 2 +-
.../clientpositive/input_testxpath4.q.out | 2 +-
.../clientpositive/insertoverwrite_bucket.q.out | 104 +
.../join_cond_pushdown_unqual4.q.out | 2 +-
.../test/results/clientpositive/lineage3.q.out | 34 +-
.../list_bucket_query_oneskew_3.q.out | 6 +-
.../clientpositive/macro_duplicate.q.out | 56 +
.../results/clientpositive/multi_insert.q.out | 8 +-
.../clientpositive/multi_insert_gby.q.out | 2 +-
.../multi_insert_lateral_view.q.out | 4 +-
...i_insert_move_tasks_share_dependencies.q.out | 360 +-
.../test/results/clientpositive/null_cast.q.out | 6 +-
.../results/clientpositive/orc_file_dump.q.out | 447 +
.../clientpositive/orc_predicate_pushdown.q.out | 36 +-
ql/src/test/results/clientpositive/pcr.q.out | 12 +-
.../results/clientpositive/ppd_gby_join.q.out | 4 +-
.../test/results/clientpositive/ppd_join.q.out | 4 +-
.../test/results/clientpositive/ppd_join2.q.out | 4 +-
.../test/results/clientpositive/ppd_join3.q.out | 6 +-
.../clientpositive/ppd_outer_join4.q.out | 2 +-
.../results/clientpositive/ppd_transform.q.out | 12 +-
ql/src/test/results/clientpositive/ptf.q.out | 27 +-
.../clientpositive/rand_partitionpruner3.q.out | 12 +-
.../clientpositive/selectDistinctStar.q.out | 8 +-
.../clientpositive/select_unquote_not.q.out | 8 +-
.../results/clientpositive/show_functions.q.out | 1 +
.../spark/dynamic_rdd_cache.q.out | 6 +-
.../spark/filter_join_breaktask.q.out | 12 +-
.../spark/groupby_multi_single_reducer2.q.out | 2 +-
.../spark/groupby_multi_single_reducer3.q.out | 12 +-
.../spark/groupby_sort_1_23.q.out | 90 +-
.../spark/groupby_sort_skew_1_23.q.out | 90 +-
.../spark/join_cond_pushdown_unqual4.q.out | 2 +-
.../clientpositive/spark/multi_insert.q.out | 8 +-
.../clientpositive/spark/multi_insert_gby.q.out | 2 +-
.../spark/multi_insert_lateral_view.q.out | 4 +-
...i_insert_move_tasks_share_dependencies.q.out | 536 +-
.../test/results/clientpositive/spark/pcr.q.out | 12 +-
.../clientpositive/spark/ppd_gby_join.q.out | 4 +-
.../results/clientpositive/spark/ppd_join.q.out | 4 +-
.../clientpositive/spark/ppd_join2.q.out | 4 +-
.../clientpositive/spark/ppd_join3.q.out | 6 +-
.../clientpositive/spark/ppd_outer_join4.q.out | 2 +-
.../clientpositive/spark/ppd_transform.q.out | 12 +-
.../test/results/clientpositive/spark/ptf.q.out | 17 +-
.../spark/union_remove_6_subq.q.out | 22 +-
.../spark/vector_cast_constant.q.java1.7.out | 41 +-
.../clientpositive/spark/vectorization_0.q.out | 2 +-
.../clientpositive/spark/vectorization_13.q.out | 4 +-
.../clientpositive/spark/vectorization_15.q.out | 2 +-
.../clientpositive/spark/vectorization_17.q.out | 2 +-
.../spark/vectorization_short_regress.q.out | 40 +-
.../clientpositive/spark/vectorized_case.q.out | 2 +-
.../clientpositive/spark/vectorized_ptf.q.out | 20 +-
.../test/results/clientpositive/structin.q.out | 110 +
.../clientpositive/tez/explainuser_1.q.out | 71 +-
.../tez/filter_join_breaktask.q.out | 12 +-
.../clientpositive/tez/orc_ppd_basic.q.out | 701 +
.../test/results/clientpositive/tez/ptf.q.out | 15 +-
.../clientpositive/tez/selectDistinctStar.q.out | 8 +-
.../tez/vector_cast_constant.q.java1.7.out | 21 +-
.../tez/vector_char_mapjoin1.q.out | 1 +
.../clientpositive/tez/vector_decimal_udf.q.out | 24 +-
.../tez/vector_mr_diff_schema_alias.q.out | 2 +-
.../tez/vector_varchar_mapjoin1.q.out | 1 +
.../clientpositive/tez/vectorization_0.q.out | 2 +-
.../clientpositive/tez/vectorization_13.q.out | 4 +-
.../clientpositive/tez/vectorization_15.q.out | 2 +-
.../clientpositive/tez/vectorization_17.q.out | 2 +-
.../clientpositive/tez/vectorization_7.q.out | 4 +-
.../clientpositive/tez/vectorization_8.q.out | 4 +-
.../tez/vectorization_short_regress.q.out | 40 +-
.../clientpositive/tez/vectorized_case.q.out | 2 +-
.../clientpositive/tez/vectorized_casts.q.out | 99 +-
.../clientpositive/tez/vectorized_ptf.q.out | 19 +-
.../results/clientpositive/udf_bround.q.out | 119 +
.../results/clientpositive/udf_inline.q.out | 8 +-
.../clientpositive/udf_isnull_isnotnull.q.out | 2 +-
ql/src/test/results/clientpositive/udf_or.q.out | 4 +-
.../test/results/clientpositive/udf_size.q.out | 2 +-
.../results/clientpositive/udf_struct.q.out | 2 +-
.../test/results/clientpositive/udf_union.q.out | 2 +-
.../clientpositive/union_remove_6_subq.q.out | 34 +-
.../clientpositive/unionall_unbalancedppd.q.out | 14 +-
.../results/clientpositive/vector_bround.q.out | 86 +
.../vector_cast_constant.q.java1.7.out | 32 +-
.../clientpositive/vector_char_mapjoin1.q.out | 1 +
.../clientpositive/vector_decimal_udf.q.out | 24 +-
.../vector_mr_diff_schema_alias.q.out | 2 +-
.../vector_varchar_mapjoin1.q.out | 1 +
.../clientpositive/vectorization_0.q.out | 2 +-
.../clientpositive/vectorization_13.q.out | 4 +-
.../clientpositive/vectorization_15.q.out | 2 +-
.../clientpositive/vectorization_17.q.out | 2 +-
.../clientpositive/vectorization_7.q.out | 4 +-
.../clientpositive/vectorization_8.q.out | 4 +-
.../vectorization_short_regress.q.out | 40 +-
.../clientpositive/vectorized_case.q.out | 9 +-
.../clientpositive/vectorized_casts.q.out | 66 +-
.../results/clientpositive/vectorized_ptf.q.out | 67 +-
.../apache/hadoop/hive/serde2/WriteBuffers.java | 2 +-
.../serde2/avro/AvroLazyObjectInspector.java | 19 +-
.../hadoop/hive/serde2/avro/InstanceCache.java | 17 +-
.../ObjectInspectorConverters.java | 31 +-
.../objectinspector/ObjectInspectorFactory.java | 68 +-
.../objectinspector/ObjectInspectorUtils.java | 19 +
.../ReflectionStructObjectInspector.java | 60 +-
.../SettableUnionObjectInspector.java | 4 +-
.../StandardConstantStructObjectInspector.java | 51 +
.../StandardStructObjectInspector.java | 7 +-
.../StandardUnionObjectInspector.java | 4 +-
.../ThriftUnionObjectInspector.java | 44 +-
.../hive/serde2/typeinfo/TypeInfoUtils.java | 2 +-
.../avro/TestAvroLazyObjectInspector.java | 59 +
.../TestObjectInspectorConverters.java | 89 +-
.../TestReflectionObjectInspectors.java | 71 +-
.../TestThriftObjectInspectors.java | 10 +-
.../hive/service/cli/CLIServiceUtils.java | 7 -
.../cli/operation/LogDivertAppender.java | 223 +-
.../service/cli/operation/OperationManager.java | 17 +-
.../service/cli/session/SessionManager.java | 42 +-
.../cli/thrift/ThriftBinaryCLIService.java | 1 -
.../apache/hive/service/server/HiveServer2.java | 74 +-
.../session/TestPluggableHiveSessionImpl.java | 55 +
shims/0.20S/pom.xml | 8 +-
.../hadoop/hive/shims/Hadoop20SShims.java | 26 +-
shims/0.23/pom.xml | 21 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 40 +-
shims/common/pom.xml | 17 +-
.../hadoop/hive/shims/HiveEventCounter.java | 102 -
.../hive/spark/client/SparkClientImpl.java | 4 +
.../src/test/resources/log4j.properties | 23 -
spark-client/src/test/resources/log4j2.xml | 39 +
storage-api/pom.xml | 7 -
.../hadoop/hive/common/type/HiveDecimal.java | 1 +
testutils/ptest2/pom.xml | 32 +-
.../ptest2/src/main/resources/log4j.properties | 37 -
testutils/ptest2/src/main/resources/log4j2.xml | 80 +
412 files changed, 30244 insertions(+), 3537 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/common/src/main/resources/hive-log4j2.xml
----------------------------------------------------------------------
diff --cc common/src/main/resources/hive-log4j2.xml
index 0000000,31b8fcc..6cad3fe
mode 000000,100644..100644
--- a/common/src/main/resources/hive-log4j2.xml
+++ b/common/src/main/resources/hive-log4j2.xml
@@@ -1,0 -1,111 +1,111 @@@
+ <?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <Configuration status="info" strict="true" name="HiveLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">DRFA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.log.file">hive.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
- <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
++ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t(%x)]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File> -->
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
- <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
++ <PatternLayout pattern="%d{ISO8601} %-5p [%t(%x)]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <!-- Rollover at mignight (interval = 1 means daily) -->
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <!-- 30-day backup -->
+ <!-- <DefaultRolloverStrategy max="30"/> -->
+ </RollingFile>
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+ </Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/data/conf/hive-site.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/itests/qtest/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --cc itests/src/test/resources/testconfiguration.properties
index 377ddb0,bed621d..96d3029
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@@ -329,9 -329,9 +329,10 @@@ minitez.query.files=bucket_map_join_tez
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
mapjoin_decimal.q,\
- lvj_mapjoin.q, \
+ lvj_mapjoin.q,\
+ llapdecider.q,\
mrr.q,\
+ orc_ppd_basic.q,\
tez_bmj_schema_evolution.q,\
tez_dml.q,\
tez_fsstat.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --cc itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index c9ed0d4,3fae0ba..f298ee8
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@@ -1836,8 -1816,7 +1835,8 @@@ public class QTestUtil
{
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
- qt[i] = new QTestUtil(
- resDir, logDir, MiniClusterType.none, null, null, "0.20", "", false);
- qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", defaultInitScript, defaultCleanupScript);
++ qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
++ defaultInitScript, defaultCleanupScript, false);
qt[i].addFile(qfiles[i]);
qt[i].clearTestSideEffects();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --cc packaging/src/main/assembly/bin.xml
index 539fcac,0fa6af8..f0c96ed
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@@ -341,26 -323,21 +342,26 @@@
<files>
<file>
- <source>${project.parent.basedir}/common/src/main/resources/hive-log4j.properties</source>
+ <source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-log4j.properties.template</destName>
+ <destName>hive-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j.properties</source>
+ <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-exec-log4j.properties.template</destName>
+ <destName>hive-exec-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j.properties</source>
+ <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>beeline-log4j.properties.template</destName>
+ <destName>beeline-log4j2.xml.template</destName>
</file>
<file>
+ <source>${project.parent.basedir}/llap-server/src/main/resources/llap-daemon-log4j.properties</source>
+ <outputDirectory>conf</outputDirectory>
+ <destName>llap-daemon-log4j.properties.template</destName>
+ </file>
+ <file>
<source>${project.parent.basedir}/hcatalog/README.txt</source>
<outputDirectory>hcatalog/share/doc/hcatalog</outputDirectory>
</file>
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index eed7000,fd6d2ad..380fd4e
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@@ -52,9 -53,9 +53,10 @@@ import org.apache.hadoop.hive.ql.io.Aci
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
+import org.apache.hadoop.hive.ql.io.LlapWrappableInputFormatInterface;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader;
+ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.Context;
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
@@@ -716,6 -726,7 +728,8 @@@ public class OrcInputFormat implement
LOG.error("Failed to get files with ID; using regular API", t);
}
}
++
+ // Fall back to regular API and create states without ID.
List<FileStatus> children = SHIMS.listLocatedStatus(fs, base, AcidUtils.hiddenFileFilter);
List<HdfsFileStatusWithId> result = new ArrayList<>(children.size());
for (FileStatus child : children) {
@@@ -737,9 -749,10 +752,10 @@@
private final TreeMap<Long, BlockLocation> locations;
private final FileInfo fileInfo;
private List<StripeInformation> stripes;
- private ReaderImpl.FileMetaInfo fileMetaInfo;
- private Metadata metadata;
+ private FileMetaInfo fileMetaInfo;
+ private List<StripeStatistics> stripeStats;
private List<OrcProto.Type> types;
+ private boolean[] includedCols;
private final boolean isOriginal;
private final List<DeltaMetaData> deltas;
private final boolean hasBase;
@@@ -750,10 -763,11 +766,11 @@@
public SplitGenerator(SplitInfo splitInfo) throws IOException {
this.context = splitInfo.context;
this.fs = splitInfo.fs;
- this.fileWithId = splitInfo.file;
- this.blockSize = fileWithId.getFileStatus().getBlockSize();
+ this.fileWithId = splitInfo.fileWithId;
+ this.file = this.fileWithId.getFileStatus();
+ this.blockSize = this.file.getBlockSize();
this.fileInfo = splitInfo.fileInfo;
- locations = SHIMS.getLocationsWithOffset(fs, file);
+ locations = SHIMS.getLocationsWithOffset(fs, fileWithId.getFileStatus());
this.isOriginal = splitInfo.isOriginal;
this.deltas = splitInfo.deltas;
this.hasBase = splitInfo.hasBase;
@@@ -951,15 -972,12 +974,12 @@@
}
private void populateAndCacheStripeDetails() throws IOException {
- Reader orcReader = OrcFile.createReader(file.getPath(),
+ Reader orcReader = OrcFile.createReader(fileWithId.getFileStatus().getPath(),
OrcFile.readerOptions(context.conf).filesystem(fs));
- List<String> projCols = Lists.newArrayList(
- ColumnProjectionUtils.getReadColumnNames(context.conf));
- projColsUncompressedSize = orcReader.getRawDataSizeOfColumns(projCols);
if (fileInfo != null) {
stripes = fileInfo.stripeInfos;
+ stripeStats = fileInfo.stripeStats;
fileMetaInfo = fileInfo.fileMetaInfo;
- metadata = fileInfo.metadata;
types = fileInfo.types;
writerVersion = fileInfo.writerVersion;
// For multiple runs, in case sendSplitsInFooter changes
@@@ -977,12 -996,27 +997,28 @@@
((ReaderImpl) orcReader).getFileMetaInfo() : null;
if (context.cacheStripeDetails) {
// Populate into cache.
- Context.footerCache.put(file.getPath(),
- new FileInfo(file.getModificationTime(), file.getLen(), stripes,
- metadata, types, fileMetaInfo, writerVersion));
+ Context.footerCache.put(fileWithId.getFileStatus().getPath(),
+ new FileInfo(fileWithId.getFileStatus().getModificationTime(),
+ fileWithId.getFileStatus().getLen(), stripes,
+ stripeStats, types, fileMetaInfo, writerVersion));
}
}
+ includedCols = genIncludedColumns(types, context.conf, isOriginal);
+ projColsUncompressedSize = computeProjectionSize(orcReader, includedCols, isOriginal);
+ }
+
+ private long computeProjectionSize(final Reader orcReader, final boolean[] includedCols,
+ final boolean isOriginal) {
+ final int rootIdx = getRootColumn(isOriginal);
+ List<Integer> internalColIds = Lists.newArrayList();
+ if (includedCols != null) {
+ for (int i = 0; i < includedCols.length; i++) {
+ if (includedCols[i]) {
+ internalColIds.add(rootIdx + i);
+ }
+ }
+ }
+ return orcReader.getRawDataSizeFromColIndices(internalColIds);
}
private boolean isStripeSatisfyPredicate(StripeStatistics stripeStatistics,
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
index dfe042d,cc03df7..33513bc
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
@@@ -177,10 -180,4 +181,9 @@@ public class OrcSplit extends FileSpli
public Long getFileId() {
return fileId;
}
+
+ @Override
+ public long getColumnarProjectionSize() {
+ return projColsUncompressedSize;
+ }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 2fbc732,c990d85..4ff1e28
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@@ -497,9 -469,38 +500,38 @@@ public class ReaderImpl implements Read
int footerBufferSize = footerBuffer.limit() - footerBuffer.position() - metadataSize;
footerBuffer.limit(position + metadataSize);
- InputStream instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
+ InputStream instream = InStream.create(null, "metadata", Lists.<DiskRange>newArrayList(
new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
- this.metadata = OrcProto.Metadata.parseFrom(instream);
+ CodedInputStream in = CodedInputStream.newInstance(instream);
+ int msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT;
+ OrcProto.Metadata meta = null;
+ do {
+ try {
+ in.setSizeLimit(msgLimit);
+ meta = OrcProto.Metadata.parseFrom(in);
+ } catch (InvalidProtocolBufferException e) {
+ if (e.getMessage().contains("Protocol message was too large")) {
+ LOG.warn("Metadata section is larger than " + msgLimit + " bytes. Increasing the max" +
+ " size of the coded input stream." );
+
+ msgLimit = msgLimit << 1;
+ if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) {
+ LOG.error("Metadata section exceeds max protobuf message size of " +
+ PROTOBUF_MESSAGE_MAX_LIMIT + " bytes.");
+ throw e;
+ }
+
+ // we must have failed in the middle of reading instream and instream doesn't support
+ // resetting the stream
- instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
++ instream = InStream.create(null, "metadata", Lists.<DiskRange>newArrayList(
+ new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
+ in = CodedInputStream.newInstance(instream);
+ } else {
+ throw e;
+ }
+ }
+ } while (meta == null);
+ this.metadata = meta;
footerBuffer.position(position + metadataSize);
footerBuffer.limit(position + metadataSize + footerBufferSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
diff --cc ql/src/main/resources/hive-exec-log4j2.xml
index 0000000,8b520a2..ef0747a
mode 000000,100644..100644
--- a/ql/src/main/resources/hive-exec-log4j2.xml
+++ b/ql/src/main/resources/hive-exec-log4j2.xml
@@@ -1,0 -1,110 +1,110 @@@
+ <?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <Configuration status="info" strict="true" name="HiveExecLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">DEBUG</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">FA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.query.id">hadoop</Property>
+ <Property name="hive.log.file">${sys:hive.query.id}.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
- <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
++ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t(%x)]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
- <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
++ <PatternLayout pattern="%d{ISO8601} %-5p [%t(%x)]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File>
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <!-- <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <DefaultRolloverStrategy max="30"/>
+ </RollingFile> -->
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+ </Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorization_15.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorization_17.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index 2db6a19,3d17aba..bff5143
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@@ -2119,10 -2119,9 +2119,10 @@@ STAGE PLANS
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
Reducer 3
+ Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vector_decimal_udf.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vector_mr_diff_schema_alias.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_13.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_15.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_17.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_7.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_8.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
index 6204d4c,59b457a..08d5cb5
--- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
@@@ -2119,10 -2119,9 +2119,10 @@@ STAGE PLANS
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
- Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
Reducer 3
+ Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/ql/src/test/results/clientpositive/vectorized_ptf.q.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88248fb6/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
[21/50] [abbrv] hive git commit: HIVE-11317 - ACID: Improve
transaction Abort logic due to timeout (Eugene Koifman,
reviewed by Alan Gates)
Posted by se...@apache.org.
HIVE-11317 - ACID: Improve transaction Abort logic due to timeout (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/97a6cd35
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/97a6cd35
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/97a6cd35
Branch: refs/heads/llap
Commit: 97a6cd35a444315885008f11c20c7c28249bd42c
Parents: e8329ee
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Sat Aug 15 10:22:55 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Sat Aug 15 10:22:55 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +
.../deployers/config/hive/hive-site.mysql.xml | 22 +++
.../hive/hcatalog/streaming/TestStreaming.java | 54 ++++++-
.../hadoop/hive/metastore/HiveMetaStore.java | 18 +++
.../hive/metastore/HouseKeeperService.java | 39 +++++
.../hadoop/hive/metastore/txn/TxnHandler.java | 153 +++++++++++++------
.../hive/metastore/txn/TestTxnHandler.java | 7 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 15 +-
.../hive/ql/txn/AcidHouseKeeperService.java | 104 +++++++++++++
.../hive/ql/txn/compactor/CompactorMR.java | 6 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 1 +
.../hadoop/hive/ql/txn/compactor/Worker.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnCommands.java | 21 +++
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 1 +
.../hive/ql/lockmgr/TestDbTxnManager.java | 35 +++--
15 files changed, 421 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 730f5be..9a6781b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1507,6 +1507,10 @@ public class HiveConf extends Configuration {
HIVE_COMPACTOR_CLEANER_RUN_INTERVAL("hive.compactor.cleaner.run.interval", "5000ms",
new TimeValidator(TimeUnit.MILLISECONDS), "Time between runs of the cleaner thread"),
+ HIVE_TIMEDOUT_TXN_REAPER_START("hive.timedout.txn.reaper.start", "100s",
+ new TimeValidator(TimeUnit.MILLISECONDS), "Time delay of 1st reaper run after metastore start"),
+ HIVE_TIMEDOUT_TXN_REAPER_INTERVAL("hive.timedout.txn.reaper.interval", "180s",
+ new TimeValidator(TimeUnit.MILLISECONDS), "Time interval describing how often the reaper runs"),
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true,
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mysql.xml
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mysql.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mysql.xml
index 70ccc31..b6f1ab7 100644
--- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mysql.xml
+++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mysql.xml
@@ -62,6 +62,28 @@
<name>hive.exec.dynamic.partition.mode</name>
<value>nonstrict</value>
</property>
+ <property>
+ <name>hive.compactor.initiator.on</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.compactor.worker.threads</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>hive.timedout.txn.reaper.start</name>
+ <value>2s</value>
+ </property>
+<!-- <property>
+ <name>hive.txn.timeout</name>
+ <value>60s</value>
+ </property>
+ -->
+ <property>
+ <name>hive.timedout.txn.reaper.interval</name>
+ <value>30s</value>
+ </property>
+
<!--end ACID related properties-->
<!--
<property>
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index c0af533..c28d4aa 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -18,7 +18,6 @@
package org.apache.hive.hcatalog.streaming;
-import junit.framework.Assert;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
@@ -36,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
+import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.InputFormat;
@@ -51,6 +52,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.thrift.TException;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -67,6 +69,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
public class TestStreaming {
@@ -301,6 +304,55 @@ public class TestStreaming {
connection.close();
}
+ /**
+ * check that transactions that have not heartbeated and timedout get properly aborted
+ * @throws Exception
+ */
+ @Test
+ public void testTimeOutReaper() throws Exception {
+ HiveEndPoint endPt = new HiveEndPoint(metaStoreURI, dbName2, tblName2, null);
+ DelimitedInputWriter writer = new DelimitedInputWriter(fieldNames2,",", endPt);
+ StreamingConnection connection = endPt.newConnection(false, null);
+
+ TransactionBatch txnBatch = connection.fetchTransactionBatch(5, writer);
+ txnBatch.beginNextTransaction();
+ conf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS);
+ //ensure txn timesout
+ conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS);
+ AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService();
+ houseKeeperService.start(conf);
+ while(houseKeeperService.getIsAliveCounter() <= Integer.MIN_VALUE) {
+ Thread.sleep(100);//make sure it has run at least once
+ }
+ houseKeeperService.stop();
+ try {
+ //should fail because the TransactionBatch timed out
+ txnBatch.commit();
+ }
+ catch(TransactionError e) {
+ Assert.assertTrue("Expected aborted transaction", e.getCause() instanceof TxnAbortedException);
+ }
+ txnBatch.close();
+ txnBatch = connection.fetchTransactionBatch(10, writer);
+ txnBatch.beginNextTransaction();
+ txnBatch.commit();
+ txnBatch.beginNextTransaction();
+ int lastCount = houseKeeperService.getIsAliveCounter();
+ houseKeeperService.start(conf);
+ while(houseKeeperService.getIsAliveCounter() <= lastCount) {
+ Thread.sleep(100);//make sure it has run at least once
+ }
+ houseKeeperService.stop();
+ try {
+ //should fail because the TransactionBatch timed out
+ txnBatch.commit();
+ }
+ catch(TransactionError e) {
+ Assert.assertTrue("Expected aborted transaction", e.getCause() instanceof TxnAbortedException);
+ }
+ txnBatch.close();
+ connection.close();
+ }
@Test
public void testTransactionBatchEmptyAbort() throws Exception {
// 1) to partitioned table
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 65117c4..ae500bf 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -6225,6 +6225,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
startCompactorInitiator(conf);
startCompactorWorkers(conf);
startCompactorCleaner(conf);
+ startHouseKeeperService(conf);
} catch (Throwable e) {
LOG.error("Failure when starting the compactor, compactions may not happen, " +
StringUtils.stringifyException(e));
@@ -6284,4 +6285,21 @@ public class HiveMetaStore extends ThriftHiveMetastore {
thread.init(new AtomicBoolean(), new AtomicBoolean());
thread.start();
}
+ private static void startHouseKeeperService(HiveConf conf) throws Exception {
+ if(!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_INITIATOR_ON)) {
+ return;
+ }
+ Class c = Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService");
+ //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop()
+ //should be called form it
+ HouseKeeperService houseKeeper = (HouseKeeperService)c.newInstance();
+ try {
+ houseKeeper.start(conf);
+ }
+ catch (Exception ex) {
+ LOG.fatal("Failed to start " + houseKeeper.getClass() +
+ ". The system will not handle " + houseKeeper.getServiceDescription() +
+ ". Root Cause: ", ex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java
new file mode 100644
index 0000000..eb4ea93
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HouseKeeperService.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ * Runs arbitrary background logic inside the metastore service.
+ */
+@InterfaceAudience.LimitedPrivate({"Hive"})
+@InterfaceStability.Evolving
+public interface HouseKeeperService {
+ public void start(HiveConf hiveConf) throws Exception;
+ /**
+ * Should perform orderly shutdown
+ */
+ public void stop();
+ /**
+ * Returns short description of services this module provides.
+ */
+ public String getServiceDescription();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 88e007c..795b2d9 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -46,6 +46,11 @@ import java.util.concurrent.TimeUnit;
/**
* A handler to answer transaction related calls that come into the metastore
* server.
+ *
+ * Note on log messages: Please include txnid:X and lockid info
+ * {@link org.apache.hadoop.hive.common.JavaUtils#lockIdToString(long)} in all messages.
+ * The txnid:X and lockid:Y matches how Thrift object toString() methods are generated,
+ * so keeping the format consistent makes grep'ing the logs much easier.
*/
public class TxnHandler {
// Compactor states
@@ -212,7 +217,6 @@ public class TxnHandler {
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
- timeOutTxns(dbConn);
stmt = dbConn.createStatement();
String s = "select ntxn_next - 1 from NEXT_TXN_ID";
LOG.debug("Going to execute query <" + s + ">");
@@ -463,8 +467,6 @@ public class TxnHandler {
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
long extLockId = rqst.getLockid();
- // Clean up timed out locks
- timeOutLocks(dbConn);
// Heartbeat on the lockid first, to assure that our lock is still valid.
// Then look up the lock info (hopefully in the cache). If these locks
@@ -1361,8 +1363,6 @@ public class TxnHandler {
// and prevent other operations (such as committing transactions, showing locks,
// etc.) that should not interfere with this one.
synchronized (lockLock) {
- // Clean up timed out locks before we attempt to acquire any.
- timeOutLocks(dbConn);
Statement stmt = null;
try {
stmt = dbConn.createStatement();
@@ -1732,15 +1732,22 @@ public class TxnHandler {
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
if (!rs.next()) {
+ s = "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_TXNID = " + txnid;
+ ResultSet rs2 = stmt.executeQuery(s);
+ boolean alreadyCommitted = rs2.next() && rs2.getInt(1) > 0;
LOG.debug("Going to rollback");
dbConn.rollback();
+ if(alreadyCommitted) {
+ //makes the message more informative - helps to find bugs in client code
+ throw new NoSuchTxnException("Transaction " + JavaUtils.txnIdToString(txnid) + " is already committed.");
+ }
throw new NoSuchTxnException("No such transaction " + JavaUtils.txnIdToString(txnid));
}
if (rs.getString(1).charAt(0) == TXN_ABORTED) {
LOG.debug("Going to rollback");
dbConn.rollback();
throw new TxnAbortedException("Transaction " + JavaUtils.txnIdToString(txnid) +
- " already aborted");
+ " already aborted");//todo: add time of abort, which is not currently tracked
}
s = "update TXNS set txn_last_heartbeat = " + now +
" where txn_id = " + txnid;
@@ -1802,61 +1809,121 @@ public class TxnHandler {
}
}
- // Clean time out locks from the database. This does a commit,
+ // Clean time out locks from the database not associated with a transactions, i.e. locks
+ // for read-only autoCommit=true statements. This does a commit,
// and thus should be done before any calls to heartbeat that will leave
// open transactions.
- private void timeOutLocks(Connection dbConn) throws SQLException, MetaException {
- long now = getDbTime(dbConn);
+ private void timeOutLocks(Connection dbConn) {
Statement stmt = null;
try {
+ long now = getDbTime(dbConn);
stmt = dbConn.createStatement();
// Remove any timed out locks from the table.
String s = "delete from HIVE_LOCKS where hl_last_heartbeat < " +
- (now - timeout);
+ (now - timeout) + " and (hl_txnid = 0 or hl_txnid is NULL)";//when txnid is > 0, the lock is
+ //associated with a txn and is handled by performTimeOuts()
+ //want to avoid expiring locks for a txn w/o expiring the txn itself
LOG.debug("Going to execute update <" + s + ">");
- stmt.executeUpdate(s);
+ int deletedLocks = stmt.executeUpdate(s);
+ if(deletedLocks > 0) {
+ LOG.info("Deleted " + deletedLocks + " locks from HIVE_LOCKS due to timeout");
+ }
LOG.debug("Going to commit");
dbConn.commit();
+ }
+ catch(SQLException ex) {
+ LOG.error("Failed to purge timedout locks due to: " + getMessage(ex), ex);
+ }
+ catch(Exception ex) {
+ LOG.error("Failed to purge timedout locks due to: " + ex.getMessage(), ex);
} finally {
closeStmt(stmt);
}
}
- // Abort timed out transactions. This does a commit,
- // and thus should be done before any calls to heartbeat that will leave
- // open transactions on the underlying database.
- private void timeOutTxns(Connection dbConn) throws SQLException, MetaException, RetryException {
- long now = getDbTime(dbConn);
+ /**
+ * Suppose you have a query "select a,b from T" and you want to limit the result set
+ * to the first 5 rows. The mechanism to do that differs in different DB.
+ * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the
+ * appropriately modified row limiting query.
+ */
+ private String addLimitClause(Connection dbConn, int numRows, String noSelectsqlQuery) throws MetaException {
+ DatabaseProduct prod = determineDatabaseProduct(dbConn);
+ switch (prod) {
+ case DERBY:
+ //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html
+ return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only";
+ case MYSQL:
+ //http://www.postgresql.org/docs/7.3/static/queries-limit.html
+ case POSTGRES:
+ //https://dev.mysql.com/doc/refman/5.0/en/select.html
+ return "select " + noSelectsqlQuery + " limit " + numRows;
+ case ORACLE:
+ //newer versions (12c and later) support OFFSET/FETCH
+ return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows;
+ case SQLSERVER:
+ //newer versions (2012 and later) support OFFSET/FETCH
+ //https://msdn.microsoft.com/en-us/library/ms189463.aspx
+ return "select TOP(" + numRows + ") " + noSelectsqlQuery;
+ default:
+ String msg = "Unrecognized database product name <" + prod + ">";
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+ }
+ /**
+ * This will find transactions that have timed out and abort them.
+ * Will also delete locks which are not associated with a transaction and have timed out
+ * Tries to keep transactions (against metastore db) small to reduce lock contention.
+ */
+ public void performTimeOuts() {
+ Connection dbConn = null;
Statement stmt = null;
+ ResultSet rs = null;
try {
- stmt = dbConn.createStatement();
- // Abort any timed out locks from the table.
- String s = "select txn_id from TXNS where txn_state = '" + TXN_OPEN +
- "' and txn_last_heartbeat < " + (now - timeout);
- LOG.debug("Going to execute query <" + s + ">");
- ResultSet rs = stmt.executeQuery(s);
- List<Long> deadTxns = new ArrayList<Long>();
- // Limit the number of timed out transactions we do in one pass to keep from generating a
- // huge delete statement
- do {
- deadTxns.clear();
- for (int i = 0; i < TIMED_OUT_TXN_ABORT_BATCH_SIZE && rs.next(); i++) {
- deadTxns.add(rs.getLong(1));
+ dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
+ long now = getDbTime(dbConn);
+ timeOutLocks(dbConn);
+ while(true) {
+ stmt = dbConn.createStatement();
+ String s = " txn_id from TXNS where txn_state = '" + TXN_OPEN +
+ "' and txn_last_heartbeat < " + (now - timeout);
+ s = addLimitClause(dbConn, 2500, s);
+ LOG.debug("Going to execute query <" + s + ">");
+ rs = stmt.executeQuery(s);
+ if(!rs.next()) {
+ return;//no more timedout txns
}
- // We don't care whether all of the transactions get deleted or not,
- // if some didn't it most likely means someone else deleted them in the interum
- if (deadTxns.size() > 0) abortTxns(dbConn, deadTxns);
- } while (deadTxns.size() > 0);
- LOG.debug("Going to commit");
- dbConn.commit();
- } catch (SQLException e) {
- LOG.debug("Going to rollback");
- rollbackDBConn(dbConn);
- checkRetryable(dbConn, e, "abortTxn");
- throw new MetaException("Unable to update transaction database "
- + StringUtils.stringifyException(e));
- } finally {
- closeStmt(stmt);
+ List<List<Long>> timedOutTxns = new ArrayList<>();
+ List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+ timedOutTxns.add(currentBatch);
+ do {
+ currentBatch.add(rs.getLong(1));
+ if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
+ currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+ timedOutTxns.add(currentBatch);
+ }
+ } while(rs.next());
+ close(rs, stmt, null);
+ dbConn.commit();
+ for(List<Long> batchToAbort : timedOutTxns) {
+ abortTxns(dbConn, batchToAbort);
+ dbConn.commit();
+ //todo: add TXNS.COMMENT filed and set it to 'aborted by system due to timeout'
+ LOG.info("Aborted the following transactions due to timeout: " + timedOutTxns.toString());
+ }
+ int numTxnsAborted = (timedOutTxns.size() - 1) * TIMED_OUT_TXN_ABORT_BATCH_SIZE +
+ timedOutTxns.get(timedOutTxns.size() - 1).size();
+ LOG.info("Aborted " + numTxnsAborted + " transactions due to timeout");
+ }
+ } catch (SQLException ex) {
+ LOG.warn("Aborting timedout transactions failed due to " + getMessage(ex), ex);
+ }
+ catch(MetaException e) {
+ LOG.warn("Aborting timedout transactions failed due to " + e.getMessage(), e);
+ }
+ finally {
+ close(rs, stmt, dbConn);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 6dc0bd3..59114fe 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -986,7 +986,8 @@ public class TestTxnHandler {
LockRequest req = new LockRequest(components, "me", "localhost");
LockResponse res = txnHandler.lock(req);
assertTrue(res.getState() == LockState.ACQUIRED);
- Thread.currentThread().sleep(10);
+ Thread.sleep(10);
+ txnHandler.performTimeOuts();
txnHandler.checkLock(new CheckLockRequest(res.getLockid()));
fail("Told there was a lock, when it should have timed out.");
} catch (NoSuchLockException e) {
@@ -1000,8 +1001,8 @@ public class TestTxnHandler {
long timeout = txnHandler.setTimeout(1);
try {
txnHandler.openTxns(new OpenTxnRequest(503, "me", "localhost"));
- Thread.currentThread().sleep(10);
- txnHandler.getOpenTxns();
+ Thread.sleep(10);
+ txnHandler.performTimeOuts();
GetOpenTxnsInfoResponse rsp = txnHandler.getOpenTxnsInfo();
int numAborted = 0;
for (TxnInfo txnInfo : rsp.getOpen_txns()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index c0c1b2e..4030075 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -977,11 +977,19 @@ public class Driver implements CommandProcessor {
return 10;
}
- boolean existingTxn = txnMgr.isTxnOpen();
+ boolean initiatingTransaction = false;
+ boolean readOnlyQueryInAutoCommit = false;
if((txnMgr.getAutoCommit() && haveAcidWrite()) || plan.getOperation() == HiveOperation.START_TRANSACTION ||
(!txnMgr.getAutoCommit() && startTxnImplicitly)) {
+ if(txnMgr.isTxnOpen()) {
+ throw new RuntimeException("Already have an open transaction txnid:" + txnMgr.getCurrentTxnId());
+ }
// We are writing to tables in an ACID compliant way, so we need to open a transaction
txnMgr.openTxn(userFromUGI);
+ initiatingTransaction = true;
+ }
+ else {
+ readOnlyQueryInAutoCommit = txnMgr.getAutoCommit() && plan.getOperation() == HiveOperation.QUERY && !haveAcidWrite();
}
// Set the transaction id in all of the acid file sinks
if (haveAcidWrite()) {
@@ -997,9 +1005,9 @@ public class Driver implements CommandProcessor {
For multi-stmt txns this is not sufficient and will be managed via WriteSet tracking
in the lock manager.*/
txnMgr.acquireLocks(plan, ctx, userFromUGI);
- if(!existingTxn) {
+ if(initiatingTransaction || readOnlyQueryInAutoCommit) {
//For multi-stmt txns we should record the snapshot when txn starts but
- // don't update it after that until txn completes. Thus the check for {@code existingTxn}
+ // don't update it after that until txn completes. Thus the check for {@code initiatingTransaction}
//For autoCommit=true, Read-only statements, txn is implicit, i.e. lock in the snapshot
//for each statement.
recordValidTxns();
@@ -1294,6 +1302,7 @@ public class Driver implements CommandProcessor {
}
private CommandProcessorResponse rollback(CommandProcessorResponse cpr) {
+ //console.printError(cpr.toString());
try {
releaseLocksAndCommitOrRollback(ctx.getHiveLocks(), false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java
new file mode 100644
index 0000000..d22ca8d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidHouseKeeperService.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HouseKeeperService;
+import org.apache.hadoop.hive.metastore.txn.TxnHandler;
+import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
+import org.apache.hadoop.hive.ql.lockmgr.TxnManagerFactory;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Performs background tasks for Transaction management in Hive.
+ * Runs inside Hive Metastore Service.
+ */
+public class AcidHouseKeeperService implements HouseKeeperService {
+ private static final Log LOG = LogFactory.getLog(AcidHouseKeeperService.class);
+ private ScheduledExecutorService pool = null;
+ private AtomicInteger isAliveCounter = new AtomicInteger(Integer.MIN_VALUE);
+ @Override
+ public void start(HiveConf hiveConf) throws Exception {
+ HiveTxnManager mgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(hiveConf);
+ if(!mgr.supportsAcid()) {
+ LOG.info(AcidHouseKeeperService.class.getName() + " not started since " +
+ mgr.getClass().getName() + " does not support Acid.");
+ return;//there are no transactions in this case
+ }
+ pool = Executors.newScheduledThreadPool(1, new ThreadFactory() {
+ private final AtomicInteger threadCounter = new AtomicInteger();
+ @Override
+ public Thread newThread(Runnable r) {
+ return new Thread(r, "DeadTxnReaper-" + threadCounter.getAndIncrement());
+ }
+ });
+ TimeUnit tu = TimeUnit.MILLISECONDS;
+ pool.scheduleAtFixedRate(new TimedoutTxnReaper(hiveConf, this),
+ hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, tu),
+ hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_INTERVAL, tu),
+ TimeUnit.MILLISECONDS);
+ LOG.info("Started " + this.getClass().getName() + " with delay/interval = " +
+ hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, tu) + "/" +
+ hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_INTERVAL, tu) + " " + tu);
+ }
+ @Override
+ public void stop() {
+ if(pool != null && !pool.isShutdown()) {
+ pool.shutdown();
+ }
+ pool = null;
+ }
+ @Override
+ public String getServiceDescription() {
+ return "Abort expired transactions";
+ }
+ private static final class TimedoutTxnReaper implements Runnable {
+ private final TxnHandler txnHandler;
+ private final AcidHouseKeeperService owner;
+ private TimedoutTxnReaper(HiveConf hiveConf, AcidHouseKeeperService owner) {
+ txnHandler = new TxnHandler(hiveConf);
+ this.owner = owner;
+ }
+ @Override
+ public void run() {
+ try {
+ txnHandler.performTimeOuts();
+ owner.isAliveCounter.incrementAndGet();
+ LOG.info("timeout reaper ran");
+ }
+ catch(Throwable t) {
+ LOG.fatal("Serious error in " + Thread.currentThread().getName() + ": " + t.getMessage(), t);
+ }
+ }
+ }
+
+ /**
+ * This is used for testing only. Each time the housekeeper runs, counter is incremented by 1.
+ * Starts with {@link java.lang.Integer#MIN_VALUE}
+ */
+ public int getIsAliveCounter() {
+ return isAliveCounter.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 6c77ba4..8e431b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -52,6 +53,7 @@ import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.OutputCommitter;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskAttemptContext;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.StringUtils;
@@ -183,7 +185,9 @@ public class CompactorMR {
LOG.debug("Setting minimum transaction to " + minTxn);
LOG.debug("Setting maximume transaction to " + maxTxn);
- JobClient.runJob(job).waitForCompletion();
+ RunningJob rj = JobClient.runJob(job);
+ LOG.info("Submitted " + (isMajor ? CompactionType.MAJOR : CompactionType.MINOR) + " compaction job '" + jobName + "' with jobID=" + rj.getID());
+ rj.waitForCompletion();
su.gatherStats();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index 32a9ef8..73715c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -52,6 +52,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
/**
* A class to initiate compactions. This will run in a separate thread.
+ * It's critical that there exactly 1 of these in a given warehouse.
*/
public class Initiator extends CompactorThread {
static final private String CLASS_NAME = Initiator.class.getName();
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index e164661..0548117 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -264,7 +264,7 @@ public class Worker extends CompactorThread {
sb.append(colName).append(",");
}
sb.setLength(sb.length() - 1);//remove trailing ,
- LOG.debug("running '" + sb.toString() + "'");
+ LOG.info("running '" + sb.toString() + "'");
Driver d = new Driver(conf, userName);
SessionState localSession = null;
if(SessionState.get() == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index c73621f..e13e6eb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -9,6 +9,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.orc.FileDump;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -24,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* The LockManager is not ready, but for no-concurrency straight-line path we can
@@ -388,6 +390,25 @@ public class TestTxnCommands {
int[][] updatedData = {{1,7},{3,7},{5,2},{5,3}};
Assert.assertEquals("Bulk update failed", stringifyValues(updatedData), rs);
}
+ @Test
+ public void testTimeOutReaper() throws Exception {
+ runStatementOnDriver("set autocommit false");
+ runStatementOnDriver("start transaction");
+ runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5");
+ //make sure currently running txn is considered aborted by housekeeper
+ hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS);
+ hiveConf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS);
+ AcidHouseKeeperService houseKeeperService = new AcidHouseKeeperService();
+ //this will abort the txn
+ houseKeeperService.start(hiveConf);
+ while(houseKeeperService.getIsAliveCounter() <= Integer.MIN_VALUE) {
+ Thread.sleep(100);//make sure it has run at least once
+ }
+ houseKeeperService.stop();
+ //this should fail because txn aborted due to timeout
+ CommandProcessorResponse cpr = runStatementOnDriverNegative("delete from " + Table.ACIDTBL + " where a = 5");
+ Assert.assertTrue("Actual: " + cpr.getErrorMessage(), cpr.getErrorMessage().contains("Transaction manager has aborted the transaction txnid:1"));
+ }
/**
* takes raw data and turns it into a string as if from Driver.getResults()
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 57e4fb9..58c2fca 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -131,6 +131,7 @@ public class TestTxnCommands2 {
hiveConf.setBoolVar(HiveConf.ConfVars.HIVEOPTINDEXFILTER, enablePPD);//enables ORC PPD
int[][] tableData = {{1,2},{3,4}};
runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData));
+ List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " where a > 1 order by a,b");
runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
Worker t = new Worker();
t.setThreadId((int) t.getId());
http://git-wip-us.apache.org/repos/asf/hive/blob/97a6cd35/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
index f57350d..db119e1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
@@ -29,10 +29,10 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -49,6 +49,7 @@ public class TestDbTxnManager {
private HiveConf conf = new HiveConf();
private HiveTxnManager txnMgr;
+ private AcidHouseKeeperService houseKeeperService = null;
private Context ctx;
private int nextInput;
private int nextOutput;
@@ -56,7 +57,6 @@ public class TestDbTxnManager {
HashSet<WriteEntity> writeEntities;
public TestDbTxnManager() throws Exception {
- conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 500, TimeUnit.MILLISECONDS);
TxnDbUtil.setConfValues(conf);
SessionState.start(conf);
ctx = new Context(conf);
@@ -179,14 +179,30 @@ public class TestDbTxnManager {
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
}
+
+ /**
+ * aborts timed out transactions
+ */
+ private void runReaper() throws Exception {
+ int lastCount = houseKeeperService.getIsAliveCounter();
+ houseKeeperService.start(conf);
+ while(houseKeeperService.getIsAliveCounter() <= lastCount) {
+ try {
+ Thread.sleep(100);//make sure it has run at least once
+ }
+ catch(InterruptedException ex) {
+ //...
+ }
+ }
+ houseKeeperService.stop();
+ }
@Test
public void testExceptions() throws Exception {
WriteEntity we = addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT);
QueryPlan qp = new MockQueryPlan(this);
txnMgr.acquireLocks(qp, ctx, "PeterI");
txnMgr.openTxn("NicholasII");
- Thread.sleep(1000);//let txn timeout
- txnMgr.getValidTxns();
+ runReaper();
LockException exception = null;
try {
txnMgr.commitTxn();
@@ -198,8 +214,7 @@ public class TestDbTxnManager {
Assert.assertEquals("Wrong Exception1", ErrorMsg.TXN_ABORTED, exception.getCanonicalErrorMsg());
exception = null;
txnMgr.openTxn("AlexanderIII");
- Thread.sleep(1000);
- txnMgr.getValidTxns();
+ runReaper();
try {
txnMgr.rollbackTxn();
}
@@ -213,8 +228,7 @@ public class TestDbTxnManager {
txnMgr.acquireLocks(qp, ctx, "PeterI");
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertThat("Unexpected lock count", locks.size(), is(1));
- Thread.sleep(1000);
- txnMgr.getValidTxns();
+ runReaper();
try {
txnMgr.heartbeat();
}
@@ -341,7 +355,6 @@ public class TestDbTxnManager {
Assert.assertTrue(sawException);
}
-
@Before
public void setUp() throws Exception {
TxnDbUtil.prepDb();
@@ -351,10 +364,14 @@ public class TestDbTxnManager {
nextOutput = 1;
readEntities = new HashSet<ReadEntity>();
writeEntities = new HashSet<WriteEntity>();
+ conf.setTimeVar(HiveConf.ConfVars.HIVE_TIMEDOUT_TXN_REAPER_START, 0, TimeUnit.SECONDS);
+ conf.setTimeVar(HiveConf.ConfVars.HIVE_TXN_TIMEOUT, 1, TimeUnit.MILLISECONDS);
+ houseKeeperService = new AcidHouseKeeperService();
}
@After
public void tearDown() throws Exception {
+ if(houseKeeperService != null) houseKeeperService.stop();
if (txnMgr != null) txnMgr.closeTxnManager();
TxnDbUtil.cleanDb();
}
[42/50] [abbrv] hive git commit: HIVE-11501 : Make
Connections.setReadOnly method compliant (Michal Wegrzyn via Sergey
Shelukhin)
Posted by se...@apache.org.
HIVE-11501 : Make Connections.setReadOnly method compliant (Michal Wegrzyn via Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f7ebff00
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f7ebff00
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f7ebff00
Branch: refs/heads/llap
Commit: f7ebff0057c8b16f44d03feb96f8a58e2ee848b0
Parents: 2de6fda
Author: Michał Węgrzyn <mi...@gmail.com>
Authored: Sat Aug 8 10:27:35 2015 +0200
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sun Aug 23 22:13:52 2015 -0700
----------------------------------------------------------------------
jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f7ebff00/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index bb2b695..3b7e004 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -1279,8 +1279,16 @@ public class HiveConnection implements java.sql.Connection {
@Override
public void setReadOnly(boolean readOnly) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
+ // Per JDBC spec, if the connection is closed a SQLException should be thrown.
+ if (isClosed) {
+ throw new SQLException("Connection is closed");
+ }
+ // Per JDBC spec, the request defines a hint to the driver to enable database optimizations.
+ // The read-only mode for this connection is disabled and cannot be enabled (isReadOnly always returns false).
+ // The most correct behavior is to throw only if the request tries to enable the read-only mode.
+ if(readOnly) {
+ throw new SQLException("Enabling read-only mode not supported");
+ }
}
/*
[33/50] [abbrv] hive git commit: HIVE-11594: Analyze Table for column
names with embedded spaces (Gopal V, reviewed by Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11594: Analyze Table for column names with embedded spaces (Gopal V, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/87930e2d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/87930e2d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/87930e2d
Branch: refs/heads/llap
Commit: 87930e2debaa795cc7e94f125065da567cde85b9
Parents: 0012864
Author: Gopal V <go...@apache.org>
Authored: Thu Aug 20 14:03:09 2015 -0700
Committer: Gopal V <go...@apache.org>
Committed: Thu Aug 20 14:03:13 2015 -0700
----------------------------------------------------------------------
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 6 +-
.../clientpositive/columnstats_quoting.q | 8 ++
.../clientpositive/columnstats_quoting.q.out | 114 +++++++++++++++++++
3 files changed, 125 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/87930e2d/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index a5f0a7f..8b7a2e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -175,7 +175,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
} else {
whereClause.append(" and ");
}
- whereClause.append(partKey).append(" = ").append(genPartValueString(partKey, value));
+ whereClause.append("`").append(partKey).append("` = ").append(genPartValueString(partKey, value));
}
}
@@ -308,9 +308,9 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
if (i > 0) {
rewrittenQueryBuilder.append(" , ");
}
- rewrittenQueryBuilder.append("compute_stats(");
+ rewrittenQueryBuilder.append("compute_stats(`");
rewrittenQueryBuilder.append(colNames.get(i));
- rewrittenQueryBuilder.append(" , ");
+ rewrittenQueryBuilder.append("` , ");
rewrittenQueryBuilder.append(numBitVectors);
rewrittenQueryBuilder.append(" )");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/87930e2d/ql/src/test/queries/clientpositive/columnstats_quoting.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/columnstats_quoting.q b/ql/src/test/queries/clientpositive/columnstats_quoting.q
new file mode 100644
index 0000000..1bf4f91
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/columnstats_quoting.q
@@ -0,0 +1,8 @@
+DROP TABLE IF EXISTS user_web_events;
+create temporary table user_web_events(`user id` bigint, `user name` string);
+
+explain analyze table user_web_events compute statistics for columns;
+analyze table user_web_events compute statistics for columns;
+
+explain analyze table user_web_events compute statistics for columns `user id`;
+analyze table user_web_events compute statistics for columns `user id`;
http://git-wip-us.apache.org/repos/asf/hive/blob/87930e2d/ql/src/test/results/clientpositive/columnstats_quoting.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/columnstats_quoting.q.out b/ql/src/test/results/clientpositive/columnstats_quoting.q.out
new file mode 100644
index 0000000..bd36ef0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/columnstats_quoting.q.out
@@ -0,0 +1,114 @@
+PREHOOK: query: DROP TABLE IF EXISTS user_web_events
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS user_web_events
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create temporary table user_web_events(`user id` bigint, `user name` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@user_web_events
+POSTHOOK: query: create temporary table user_web_events(`user id` bigint, `user name` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@user_web_events
+PREHOOK: query: explain analyze table user_web_events compute statistics for columns
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze table user_web_events compute statistics for columns
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: user_web_events
+ Select Operator
+ expressions: user id (type: bigint), user name (type: string)
+ outputColumnNames: user id, user name
+ Group By Operator
+ aggregations: compute_stats(user id, 16), compute_stats(user name, 16)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ sort order:
+ value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>), _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-1
+ Column Stats Work
+ Column Stats Desc:
+ Columns: user id, user name
+ Column Types: bigint, string
+ Table: default.user_web_events
+
+PREHOOK: query: analyze table user_web_events compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@user_web_events
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table user_web_events compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@user_web_events
+#### A masked pattern was here ####
+PREHOOK: query: explain analyze table user_web_events compute statistics for columns `user id`
+PREHOOK: type: QUERY
+POSTHOOK: query: explain analyze table user_web_events compute statistics for columns `user id`
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: user_web_events
+ Select Operator
+ expressions: user id (type: bigint)
+ outputColumnNames: user id
+ Group By Operator
+ aggregations: compute_stats(user id, 16)
+ mode: hash
+ outputColumnNames: _col0
+ Reduce Output Operator
+ sort order:
+ value expressions: _col0 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:string,numbitvectors:int>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: compute_stats(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-1
+ Column Stats Work
+ Column Stats Desc:
+ Columns: user id
+ Column Types: bigint
+ Table: default.user_web_events
+
+PREHOOK: query: analyze table user_web_events compute statistics for columns `user id`
+PREHOOK: type: QUERY
+PREHOOK: Input: default@user_web_events
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table user_web_events compute statistics for columns `user id`
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@user_web_events
+#### A masked pattern was here ####
[39/50] [abbrv] hive git commit: HIVE-11596: nvl(x,
y) throws NPE if type x and type y doesn't match,
rather than throwing the meaningful error (Aihua Xu, reviewed by Chao Sun)
Posted by se...@apache.org.
HIVE-11596: nvl(x, y) throws NPE if type x and type y doesn't match, rather than throwing the meaningful error (Aihua Xu, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c1eae0c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c1eae0c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c1eae0c
Branch: refs/heads/llap
Commit: 3c1eae0c44450ebbf26c88066de871ea7f479981
Parents: 1030ae7
Author: Aihua Xu <ai...@gmail.com>
Authored: Fri Aug 21 15:23:53 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 21 15:23:53 2015 -0700
----------------------------------------------------------------------
.../hive/ql/udf/generic/GenericUDFBasePad.java | 8 ++--
.../hive/ql/udf/generic/GenericUDFNvl.java | 2 +-
.../queries/clientnegative/nvl_mismatch_type.q | 20 +++++++++
.../clientnegative/char_pad_convert_fail0.q.out | 2 +-
.../clientnegative/char_pad_convert_fail1.q.out | 2 +-
.../clientnegative/char_pad_convert_fail3.q.out | 2 +-
.../clientnegative/nvl_mismatch_type.q.out | 43 ++++++++++++++++++++
7 files changed, 71 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
index 3904ca4..33839f7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
@@ -94,8 +94,8 @@ public abstract class GenericUDFBasePad extends GenericUDF {
private Converter checkTextArguments(ObjectInspector[] arguments, int i)
throws UDFArgumentException {
if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(i + 1, "Only primitive type arguments are accepted but "
- + arguments[i].getTypeName() + " is passed. as arguments");
+ throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but "
+ + arguments[i].getTypeName() + " is passed.");
}
Converter converter = ObjectInspectorConverters.getConverter((PrimitiveObjectInspector) arguments[i],
@@ -107,8 +107,8 @@ public abstract class GenericUDFBasePad extends GenericUDF {
private Converter checkIntArguments(ObjectInspector[] arguments, int i)
throws UDFArgumentException {
if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(i + 1, "Only primitive type arguments are accepted but "
- + arguments[i].getTypeName() + " is passed. as arguments");
+ throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but "
+ + arguments[i].getTypeName() + " is passed.");
}
PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory();
Converter converter;
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
index 625b466..0a16da8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
@@ -42,7 +42,7 @@ public class GenericUDFNvl extends GenericUDF{
returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
if (!(returnOIResolver.update(arguments[0]) && returnOIResolver
.update(arguments[1]))) {
- throw new UDFArgumentTypeException(2,
+ throw new UDFArgumentTypeException(1,
"The first and seconds arguments of function NLV should have the same type, "
+ "but they are different: \"" + arguments[0].getTypeName()
+ "\" and \"" + arguments[1].getTypeName() + "\"");
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/nvl_mismatch_type.q b/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
new file mode 100644
index 0000000..ace180c
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
@@ -0,0 +1,20 @@
+
+create table over1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|';
+
+load data local inpath '../../data/files/over1k' into table over1k;
+
+-- Integers
+select nvl(t, true) from over1k limit 5;
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
index d6f5502..ed2f516 100644
--- a/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
+++ b/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
@@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@over1k
-FAILED: SemanticException [Error 10016]: Line 7:29 Argument type mismatch '' '': Only primitive type arguments are accepted but array<int> is passed. as arguments
+FAILED: SemanticException [Error 10016]: Line 7:15 Argument type mismatch '3': Only primitive type arguments are accepted but array<int> is passed.
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
index 81ceeec..5f17cfb 100644
--- a/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
+++ b/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
@@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@over1k
-FAILED: SemanticException [Error 10016]: Line 7:26 Argument type mismatch '4': Only primitive type arguments are accepted but array<int> is passed. as arguments
+FAILED: SemanticException [Error 10016]: Line 7:12 Argument type mismatch '3': Only primitive type arguments are accepted but array<int> is passed.
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
index 9896eae..a25ea15 100644
--- a/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
+++ b/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
@@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@over1k
-FAILED: SemanticException [Error 10016]: Line 7:38 Argument type mismatch '4': Only primitive type arguments are accepted but uniontype<string> is passed. as arguments
+FAILED: SemanticException [Error 10016]: Line 7:12 Argument type mismatch ''Union'': Only primitive type arguments are accepted but uniontype<string> is passed.
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out b/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
new file mode 100644
index 0000000..6e88385
--- /dev/null
+++ b/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: create table over1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k
+POSTHOOK: query: create table over1k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal(4,2),
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k
+PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@over1k
+POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@over1k
+FAILED: SemanticException [Error 10016]: Line 4:14 Argument type mismatch 'true': The first and seconds arguments of function NLV should have the same type, but they are different: "tinyint" and "boolean"
[07/50] [abbrv] hive git commit: HIVE-11534: Improve
validateTableCols error message (Mohit via Xuefu)
Posted by se...@apache.org.
HIVE-11534: Improve validateTableCols error message (Mohit via Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4849cb2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4849cb2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4849cb2
Branch: refs/heads/llap
Commit: a4849cb2c5b7d1c542881e12bf0e48f73917d96c
Parents: b8f1ae1
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Fri Aug 14 06:48:19 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Fri Aug 14 06:48:19 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/TestHiveMetaStore.java | 66 ++++++++++++++++++++
.../hadoop/hive/metastore/ObjectStore.java | 7 ++-
2 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a4849cb2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 5a344bb..160667d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2966,4 +2966,70 @@ public abstract class TestHiveMetaStore extends TestCase {
};
return hookLoader;
}
+
+ public void testValidateTableCols() throws Throwable {
+
+ try {
+ String dbName = "compdb";
+ String tblName = "comptbl";
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ Database db = new Database();
+ db.setName(dbName);
+ db.setDescription("Validate Table Columns test");
+ client.createDatabase(db);
+
+ ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
+ cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
+ cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));
+
+ Table tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(tblName);
+ StorageDescriptor sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(cols);
+ sd.setCompressed(false);
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+ sd.getSerdeInfo().getParameters()
+ .put(serdeConstants.SERIALIZATION_FORMAT, "1");
+ sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName());
+ sd.setInputFormat(HiveInputFormat.class.getName());
+ sd.setOutputFormat(HiveOutputFormat.class.getName());
+ sd.setSortCols(new ArrayList<Order>());
+
+ client.createTable(tbl);
+ if (isThriftClient) {
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ List<String> expectedCols = Lists.newArrayList();
+ expectedCols.add("name");
+ ObjectStore objStore = new ObjectStore();
+ try {
+ objStore.validateTableCols(tbl, expectedCols);
+ } catch (MetaException ex) {
+ throw new RuntimeException(ex);
+ }
+
+ expectedCols.add("doesntExist");
+ boolean exceptionFound = false;
+ try {
+ objStore.validateTableCols(tbl, expectedCols);
+ } catch (MetaException ex) {
+ assertEquals(ex.getMessage(),
+ "Column doesntExist doesn't exist in table comptbl in database compdb");
+ exceptionFound = true;
+ }
+ assertTrue(exceptionFound);
+
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testValidateTableCols() failed.");
+ throw e;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a4849cb2/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index a37fbde..d165fc8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -51,6 +51,7 @@ import javax.jdo.Transaction;
import javax.jdo.datastore.DataStoreCache;
import javax.jdo.identity.IntIdentity;
+import com.google.common.annotations.VisibleForTesting;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.apache.commons.logging.Log;
@@ -6261,7 +6262,8 @@ public class ObjectStore implements RawStore, Configurable {
}
}
- private void validateTableCols(Table table, List<String> colNames) throws MetaException {
+ @VisibleForTesting
+ public void validateTableCols(Table table, List<String> colNames) throws MetaException {
List<FieldSchema> colList = table.getSd().getCols();
for (String colName : colNames) {
boolean foundCol = false;
@@ -6272,7 +6274,8 @@ public class ObjectStore implements RawStore, Configurable {
}
}
if (!foundCol) {
- throw new MetaException("Column " + colName + " doesn't exist.");
+ throw new MetaException("Column " + colName + " doesn't exist in table "
+ + table.getTableName() + " in database " + table.getDbName());
}
}
}
[16/50] [abbrv] hive git commit: HIVE-11441: No DDL allowed on table
if user accidentally set table location wrong (Daniel Dai reviewed by Thejas
Nair)
Posted by se...@apache.org.
HIVE-11441: No DDL allowed on table if user accidentally set table location wrong (Daniel Dai reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ccd0616
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ccd0616
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ccd0616
Branch: refs/heads/llap
Commit: 2ccd061691cd52ed9fa341b61590edb2c022b031
Parents: 17e95c7
Author: Daniel Dai <da...@hortonworks.com>
Authored: Fri Aug 14 15:28:44 2015 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Fri Aug 14 15:28:44 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 11 +++++++++++
.../queries/clientnegative/alter_table_wrong_location.q | 4 ++++
.../clientnegative/alter_table_wrong_location.q.out | 9 +++++++++
3 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 21625bc..9f8c756 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -23,6 +23,7 @@ import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -137,6 +138,7 @@ import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.StringUtils;
+import java.io.FileNotFoundException;
import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
@@ -1464,6 +1466,15 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
HashMap<String, String> partSpec) throws SemanticException {
String newLocation = unescapeSQLString(ast.getChild(0).getText());
+ try {
+ // To make sure host/port pair is valid, the status of the location
+ // does not matter
+ FileSystem.get(new URI(newLocation), conf).getFileStatus(new Path(newLocation));
+ } catch (FileNotFoundException e) {
+ // Only check host/port pair is valid, wheter the file exist or not does not matter
+ } catch (Exception e) {
+ throw new SemanticException("Cannot connect to namenode, please check if host/port pair for " + newLocation + " is valid", e);
+ }
addLocationToOutputs(newLocation);
AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, newLocation, partSpec);
http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/alter_table_wrong_location.q b/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
new file mode 100644
index 0000000..3721867
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
@@ -0,0 +1,4 @@
+create table testwrongloc(id int);
+
+-- Assume port 12345 is not open
+alter table testwrongloc set location "hdfs://localhost:12345/tmp/testwrongloc";
http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out b/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
new file mode 100644
index 0000000..d788d55
--- /dev/null
+++ b/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
@@ -0,0 +1,9 @@
+PREHOOK: query: create table testwrongloc(id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testwrongloc
+POSTHOOK: query: create table testwrongloc(id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testwrongloc
+#### A masked pattern was here ####
[43/50] [abbrv] hive git commit: HIVE-11176 : Caused by:
java.lang.ClassCastException:
org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct cannot be cast to
[Ljava.lang.Object; (Navis via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11176 : Caused by: java.lang.ClassCastException: org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct cannot be cast to [Ljava.lang.Object; (Navis via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a16bbd48
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a16bbd48
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a16bbd48
Branch: refs/heads/llap
Commit: a16bbd4877da11871656ed082a84d6809139740f
Parents: f7ebff0
Author: Navis Ryu <na...@apache.org>
Authored: Tue Aug 11 17:34:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 24 00:30:11 2015 -0700
----------------------------------------------------------------------
.../serde2/objectinspector/StandardStructObjectInspector.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a16bbd48/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
index c96fc2d..87a072c 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
@@ -163,8 +163,13 @@ public class StandardStructObjectInspector extends
}
// We support both List<Object> and Object[]
// so we have to do differently.
- boolean isArray = ! (data instanceof List);
+ boolean isArray = data.getClass().isArray();
if (!isArray && !(data instanceof List)) {
+ if (!warned) {
+ LOG.warn("Invalid type for struct " + data.getClass());
+ LOG.warn("ignoring similar errors.");
+ warned = true;
+ }
return data;
}
int listSize = (isArray ? ((Object[]) data).length : ((List<Object>) data)
[09/50] [abbrv] hive git commit: HIVE-11304: Migrate to Log4j2 from
Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index f156cc4..acce713 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -94,7 +94,7 @@
}
mkdir -p $DOWNLOAD_DIR
download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
- cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+ cp -f $HIVE_ROOT/data/conf/spark/log4j2.xml $BASE_DIR/spark/conf/
sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java
javac -cp ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp
jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index b33cb58..65117c4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5931,7 +5931,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// If the log4j.configuration property hasn't already been explicitly set,
// use Hive's default log4j configuration
- if (System.getProperty("log4j.configuration") == null) {
+ if (System.getProperty("log4j.configurationFile") == null) {
// NOTE: It is critical to do this here so that log4j is reinitialized
// before any of the other core hive classes are loaded
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index ad99427..df42f1a 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -17,15 +17,11 @@
*/
package org.apache.hadoop.hive.metastore.txn;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,7 +30,29 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
/**
* Tests for TxnHandler.
@@ -43,11 +61,9 @@ public class TestCompactionTxnHandler {
private HiveConf conf = new HiveConf();
private CompactionTxnHandler txnHandler;
- static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);
public TestCompactionTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
- LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
tearDown();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index f478184..6dc0bd3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -17,16 +17,11 @@
*/
package org.apache.hadoop.hive.metastore.txn;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
import java.sql.Connection;
import java.sql.SQLException;
@@ -36,21 +31,66 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnInfo;
+import org.apache.hadoop.hive.metastore.api.TxnOpenException;
+import org.apache.hadoop.hive.metastore.api.TxnState;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests for TxnHandler.
*/
public class TestTxnHandler {
static final private String CLASS_NAME = TxnHandler.class.getName();
- static final private Log LOG = LogFactory.getLog(CLASS_NAME);
+ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
private HiveConf conf = new HiveConf();
private TxnHandler txnHandler;
public TestTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
- LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration conf = ctx.getConfiguration();
+ conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
+ ctx.updateLoggers(conf);
tearDown();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index 63253c5..0fa6af8 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -275,7 +275,7 @@
<fileMode>644</fileMode>
<includes>
<include>webhcat-default.xml</include>
- <include>webhcat-log4j.properties</include>
+ <include>webhcat-log4j2.xml</include>
</includes>
<filtered>true</filtered>
<outputDirectory>hcatalog/etc/webhcat</outputDirectory>
@@ -323,19 +323,19 @@
<files>
<file>
- <source>${project.parent.basedir}/common/src/main/resources/hive-log4j.properties</source>
+ <source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-log4j.properties.template</destName>
+ <destName>hive-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j.properties</source>
+ <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-exec-log4j.properties.template</destName>
+ <destName>hive-exec-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j.properties</source>
+ <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>beeline-log4j.properties.template</destName>
+ <destName>beeline-log4j2.xml.template</destName>
</file>
<file>
<source>${project.parent.basedir}/hcatalog/README.txt</source>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0383e01..15c2805 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,8 +148,7 @@
<kryo.version>2.22</kryo.version>
<libfb303.version>0.9.2</libfb303.version>
<libthrift.version>0.9.2</libthrift.version>
- <log4j.version>1.2.16</log4j.version>
- <log4j-extras.version>1.2.17</log4j-extras.version>
+ <log4j2.version>2.3</log4j2.version>
<opencsv.version>2.3</opencsv.version>
<mockito-all.version>1.9.5</mockito-all.version>
<mina.version>2.0.0-M5</mina.version>
@@ -366,14 +365,24 @@
<version>${junit.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
@@ -584,11 +593,6 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>${xerces.version}</version>
@@ -604,11 +608,6 @@
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
</dependencies>
<build>
@@ -872,7 +871,7 @@
<!-- required for hive-exec jar path and tests which reference a jar -->
<maven.local.repository>${maven.repo.local}</maven.local.repository>
<mapred.job.tracker>local</mapred.job.tracker>
- <log4j.configuration>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j.properties</log4j.configuration>
+ <log4j.configurationFile>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.xml</log4j.configurationFile>
<log4j.debug>true</log4j.debug>
<!-- don't diry up /tmp -->
<java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index e7a8e7b..36b3433 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -112,14 +112,19 @@
<version>${javolution.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index a2cf712..82345ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -27,7 +27,6 @@ import java.lang.management.MemoryMXBean;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Enumeration;
import java.util.List;
import java.util.Properties;
@@ -57,13 +56,14 @@ import org.apache.hadoop.hive.ql.exec.PartitionKeySampler;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.io.IOPrepareCache;
+import org.apache.hadoop.hive.ql.log.NullAppender;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -88,11 +88,12 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Appender;
-import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.varia.NullAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
/**
* ExecDriver is the central class in co-ordinating execution of any map-reduce task.
@@ -687,8 +688,10 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
if (noLog) {
// If started from main(), and noLog is on, we should not output
// any logs. To turn the log on, please set -Dtest.silent=false
- BasicConfigurator.resetConfiguration();
- BasicConfigurator.configure(new NullAppender());
+ Logger logger = org.apache.logging.log4j.LogManager.getRootLogger();
+ NullAppender appender = NullAppender.createNullAppender();
+ appender.addToLogger(logger.getName(), Level.ERROR);
+ appender.start();
} else {
setupChildLog4j(conf);
}
@@ -703,10 +706,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
// print out the location of the log file for the user so
// that it's easy to find reason for local mode execution failures
- for (Appender appender : Collections.list((Enumeration<Appender>) LogManager.getRootLogger()
- .getAllAppenders())) {
+ for (Appender appender : ((org.apache.logging.log4j.core.Logger) LogManager.getRootLogger())
+ .getAppenders().values()) {
if (appender instanceof FileAppender) {
- console.printInfo("Execution log at: " + ((FileAppender) appender).getFile());
+ console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
index 6a6593c..44dfe3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
@@ -24,7 +24,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -33,8 +32,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.MapRedStats;
@@ -59,9 +56,11 @@ import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapred.TaskReport;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
public class HadoopJobExecHelper {
@@ -492,10 +491,11 @@ public class HadoopJobExecHelper {
sb.append("Logs:\n");
console.printError(sb.toString());
- for (Appender a : Collections.list((Enumeration<Appender>)
- LogManager.getRootLogger().getAllAppenders())) {
- if (a instanceof FileAppender) {
- console.printError((new Path(((FileAppender)a).getFile())).toUri().getPath());
+ for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
+ if (appender instanceof FileAppender) {
+ console.printError(((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printError(((RollingFileAppender) appender).getFileName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
index 3cb9e9c..cee0878 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.io.rcfile.stats;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -59,9 +57,11 @@ import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
/**
* PartialScanTask.
@@ -335,15 +335,15 @@ public class PartialScanTask extends Task<PartialScanWork> implements
// print out the location of the log file for the user so
// that it's easy to find reason for local mode execution failures
- for (Appender appender : Collections
- .list((Enumeration<Appender>) LogManager.getRootLogger()
- .getAllAppenders())) {
+ for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
if (appender instanceof FileAppender) {
- console.printInfo("Execution log at: "
- + ((FileAppender) appender).getFile());
+ console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
}
}
+
PartialScanWork mergeWork = new PartialScanWork(inputPaths);
DriverContext driverCxt = new DriverContext();
PartialScanTask taskExec = new PartialScanTask();
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
new file mode 100644
index 0000000..46662c4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginElement;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A log4J2 Appender that simply counts logging events in four levels:
+ * fatal, error, warn and info. The class name is used in log4j2.xml
+ */
+@Plugin(name = "HiveEventCounter", category = "Core", elementType = "appender", printObject = true)
+public class HiveEventCounter extends AbstractAppender {
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ private static final String APPENDER_NAME = "HiveEventCounter";
+ private static final int FATAL = 0;
+ private static final int ERROR = 1;
+ private static final int WARN = 2;
+ private static final int INFO = 3;
+
+ private static class EventCounts {
+ private final AtomicLongArray counts = new AtomicLongArray(4);
+
+ private void incr(int i) {
+ counts.incrementAndGet(i);
+ }
+
+ private long get(int i) {
+ return counts.get(i);
+ }
+ }
+
+ private static EventCounts counts = new EventCounts();
+
+ protected HiveEventCounter(String name, Filter filter,
+ Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+ super(name, filter, layout, ignoreExceptions);
+ }
+
+ @PluginFactory
+ public static HiveEventCounter createInstance(@PluginAttribute("name") String name,
+ @PluginAttribute("ignoreExceptions") boolean ignoreExceptions,
+ @PluginElement("Layout") Layout layout,
+ @PluginElement("Filters") Filter filter) {
+ if (name == null) {
+ name = APPENDER_NAME;
+ }
+
+ if (layout == null) {
+ layout = PatternLayout.createDefaultLayout();
+ }
+ return new HiveEventCounter(name, filter, layout, ignoreExceptions);
+ }
+
+ @InterfaceAudience.Private
+ public static long getFatal() {
+ return counts.get(FATAL);
+ }
+
+ @InterfaceAudience.Private
+ public static long getError() {
+ return counts.get(ERROR);
+ }
+
+ @InterfaceAudience.Private
+ public static long getWarn() {
+ return counts.get(WARN);
+ }
+
+ @InterfaceAudience.Private
+ public static long getInfo() {
+ return counts.get(INFO);
+ }
+
+ @VisibleForTesting
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ @VisibleForTesting
+ public void removeFromLogger(String loggerName) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.removeAppender(APPENDER_NAME);
+ context.updateLoggers();
+ }
+
+ public void append(LogEvent event) {
+ Level level = event.getLevel();
+ if (level.equals(Level.INFO)) {
+ counts.incr(INFO);
+ } else if (level.equals(Level.WARN)) {
+ counts.incr(WARN);
+ } else if (level.equals(Level.ERROR)) {
+ counts.incr(ERROR);
+ } else if (level.equals(Level.FATAL)) {
+ counts.incr(FATAL);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
new file mode 100644
index 0000000..c4cb7dd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+/**
+ * A NullAppender merely exists, it never outputs a message to any device.
+ */
+@Plugin(name = "NullAppender", category = "Core", elementType = "appender", printObject = false)
+public class NullAppender extends AbstractAppender {
+
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+
+ protected NullAppender(String name, Filter filter,
+ Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+ super(name, filter, layout, ignoreExceptions);
+ }
+
+ @PluginFactory
+ public static NullAppender createNullAppender() {
+ return new NullAppender("NullAppender", null, PatternLayout.createDefaultLayout(), true);
+ }
+
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ public void append(LogEvent event) {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
deleted file mode 100644
index 6a59d4a..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.log;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-
-import org.apache.log4j.DailyRollingFileAppender;
-
-public class PidDailyRollingFileAppender extends DailyRollingFileAppender {
-
- @Override
- public void setFile(String file) {
- RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean();
- super.setFile(file + '.' + rt.getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
new file mode 100644
index 0000000..4db10bb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.lang.management.ManagementFactory;
+
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.pattern.AbstractPatternConverter;
+import org.apache.logging.log4j.core.pattern.ArrayPatternConverter;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+
+/**
+ * FilePattern converter that converts %pid pattern to <process-id>@<hostname> information
+ * obtained at runtime.
+ *
+ * Example usage:
+ * <RollingFile name="Rolling-default" fileName="test.log" filePattern="test.log.%pid.gz">
+ *
+ * Will generate output file with name containing <process-id>@<hostname> like below
+ * test.log.95232@localhost.gz
+ */
+@Plugin(name = "PidFilePatternConverter", category = "FileConverter")
+@ConverterKeys({ "pid" })
+public class PidFilePatternConverter extends AbstractPatternConverter implements
+ ArrayPatternConverter {
+
+ /**
+ * Private constructor.
+ */
+ private PidFilePatternConverter() {
+ super("pid", "pid");
+ }
+
+ @PluginFactory
+ public static PidFilePatternConverter newInstance() {
+ return new PidFilePatternConverter();
+ }
+
+ public void format(StringBuilder toAppendTo, Object... objects) {
+ toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+ }
+
+ public void format(Object obj, StringBuilder toAppendTo) {
+ toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j.properties
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j.properties b/ql/src/main/resources/hive-exec-log4j.properties
deleted file mode 100644
index 9eaa6b6..0000000
--- a/ql/src/main/resources/hive-exec-log4j.properties
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=INFO,FA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.query.id=hadoop
-hive.log.file=${hive.query.id}.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=${hive.log.threshold}
-
-#
-# File Appender
-#
-
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,FA
-log4j.category.Datastore=ERROR,FA
-log4j.category.Datastore.Schema=ERROR,FA
-log4j.category.JPOX.Datastore=ERROR,FA
-log4j.category.JPOX.Plugin=ERROR,FA
-log4j.category.JPOX.MetaData=ERROR,FA
-log4j.category.JPOX.Query=ERROR,FA
-log4j.category.JPOX.General=ERROR,FA
-log4j.category.JPOX.Enhancer=ERROR,FA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
new file mode 100644
index 0000000..c93437c
--- /dev/null
+++ b/ql/src/main/resources/hive-exec-log4j2.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveExecLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">FA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.query.id">hadoop</Property>
+ <Property name="hive.log.file">${sys:hive.query.id}.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File>
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <!-- <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <DefaultRolloverStrategy max="30"/>
+ </RollingFile> -->
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
new file mode 100644
index 0000000..bdd837e
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hive.ql.metadata.StringAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestLog4j2Appenders {
+
+ @Before
+ public void setup() {
+ // programmatically set root logger level to INFO. By default if log4j2-test.xml is not
+ // available root logger will use ERROR log level
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration config = ctx.getConfiguration();
+ LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
+ loggerConfig.setLevel(Level.INFO);
+ ctx.updateLoggers();
+ }
+
+ @Test
+ public void testStringAppender() throws Exception {
+ // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+ Logger logger = LogManager.getRootLogger();
+ // Create a String Appender to capture log output
+ StringAppender appender = StringAppender.createStringAppender("%m");
+ appender.addToLogger(logger.getName(), Level.INFO);
+ appender.start();
+
+ // Log to the string appender
+ logger.info("Hello!");
+ logger.info(" World");
+
+ assertEquals("Hello! World", appender.getOutput());
+ appender.removeFromLogger(LogManager.getRootLogger().getName());
+ }
+
+ @Test
+ public void testHiveEventCounterAppender() throws Exception {
+ Logger logger = LogManager.getRootLogger();
+ HiveEventCounter appender = HiveEventCounter.createInstance("EventCounter", true, null, null);
+ appender.addToLogger(logger.getName(), Level.INFO);
+ appender.start();
+
+ logger.info("Test");
+ logger.info("Test");
+ logger.info("Test");
+ logger.info("Test");
+
+ logger.error("Test");
+ logger.error("Test");
+ logger.error("Test");
+
+ logger.warn("Test");
+ logger.warn("Test");
+
+ logger.fatal("Test");
+
+ // HiveEventCounter will be loaded from hive-log4j2-test.xml before tests are run. The 2 log
+ // info msgs from previous test case will also be counted along with 4 log info msgs in this
+ // test and hence we assert for 6 here
+ assertEquals(6, appender.getInfo());
+ assertEquals(3, appender.getError());
+ assertEquals(2, appender.getWarn());
+ assertEquals(1, appender.getFatal());
+ appender.removeFromLogger(LogManager.getRootLogger().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
new file mode 100644
index 0000000..17b64d6
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Log4j2 appender that writers to in-memory string object.
+ */
+@Plugin(name = "StringAppender", category = "Core", elementType = "appender", printObject = true)
+public class StringAppender
+ extends AbstractOutputStreamAppender<StringAppender.StringOutputStreamManager> {
+
+ private static final String APPENDER_NAME = "StringAppender";
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ private StringOutputStreamManager manager;
+
+ /**
+ * Instantiate a WriterAppender and set the output destination to a
+ * new {@link OutputStreamWriter} initialized with <code>os</code>
+ * as its {@link OutputStream}.
+ *
+ * @param name The name of the Appender.
+ * @param layout The layout to format the message.
+ * @param filter
+ * @param ignoreExceptions
+ * @param immediateFlush
+ * @param manager The OutputStreamManager.
+ */
+ protected StringAppender(String name,
+ Layout<? extends Serializable> layout, Filter filter,
+ boolean ignoreExceptions, boolean immediateFlush,
+ StringOutputStreamManager manager) {
+ super(name, layout, filter, ignoreExceptions, immediateFlush, manager);
+ this.manager = manager;
+ }
+
+ @PluginFactory
+ public static StringAppender createStringAppender(
+ @PluginAttribute("name") String nullablePatternString) {
+ PatternLayout layout;
+ if (nullablePatternString == null) {
+ layout = PatternLayout.createDefaultLayout();
+ } else {
+ layout = PatternLayout.createLayout(nullablePatternString, configuration,
+ null, null, true, false, null, null);
+ }
+
+ return new StringAppender(APPENDER_NAME, layout, null, false, true,
+ new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", layout));
+ }
+
+ @VisibleForTesting
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ @VisibleForTesting
+ public void removeFromLogger(String loggerName) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.removeAppender(APPENDER_NAME);
+ context.updateLoggers();
+ }
+
+ public String getOutput() {
+ manager.flush();
+ return new String(manager.getStream().toByteArray());
+ }
+
+ public void reset() {
+ manager.reset();
+ }
+
+ protected static class StringOutputStreamManager extends OutputStreamManager {
+ ByteArrayOutputStream stream;
+
+ protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+ Layout<?> layout) {
+ super(os, streamName, layout);
+ stream = os;
+ }
+
+ public ByteArrayOutputStream getStream() {
+ return stream;
+ }
+
+ public void reset() {
+ stream.reset();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 99fbd5d..1e2feaa 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.metadata;
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -29,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import junit.framework.TestCase;
-
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -56,15 +53,19 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.WriterAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.junit.Assert;
import com.google.common.collect.ImmutableMap;
+import junit.framework.TestCase;
+
/**
* TestHive.
*
@@ -248,36 +249,39 @@ public class TestHive extends TestCase {
* @throws Throwable
*/
public void testMetaStoreApiTiming() throws Throwable {
- // set log level to DEBUG, as this is logged at debug level
- Logger logger = Logger.getLogger("hive.ql.metadata.Hive");
- Level origLevel = logger.getLevel();
- logger.setLevel(Level.DEBUG);
-
- // create an appender to capture the logs in a string
- StringWriter writer = new StringWriter();
- WriterAppender appender = new WriterAppender(new PatternLayout(), writer);
+ // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+ Logger logger = LogManager.getLogger("hive.ql.metadata.Hive");
+ Level oldLevel = logger.getLevel();
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration config = ctx.getConfiguration();
+ LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+ loggerConfig.setLevel(Level.DEBUG);
+ ctx.updateLoggers();
+
+ // Create a String Appender to capture log output
+ StringAppender appender = StringAppender.createStringAppender("%m");
+ appender.addToLogger(logger.getName(), Level.DEBUG);
+ appender.start();
try {
- logger.addAppender(appender);
-
hm.clearMetaCallTiming();
hm.getAllDatabases();
hm.dumpAndClearMetaCallTiming("test");
- String logStr = writer.toString();
+ String logStr = appender.getOutput();
String expectedString = "getAllDatabases_()=";
Assert.assertTrue(logStr + " should contain <" + expectedString,
logStr.contains(expectedString));
// reset the log buffer, verify new dump without any api call does not contain func
- writer.getBuffer().setLength(0);
+ appender.reset();
hm.dumpAndClearMetaCallTiming("test");
- logStr = writer.toString();
+ logStr = appender.getOutput();
Assert.assertFalse(logStr + " should not contain <" + expectedString,
logStr.contains(expectedString));
-
} finally {
- logger.setLevel(origLevel);
- logger.removeAppender(appender);
+ loggerConfig.setLevel(oldLevel);
+ ctx.updateLoggers();
+ appender.removeFromLogger(logger.getName());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
index 9d64b10..876ade8 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
@@ -18,9 +18,6 @@
package org.apache.hive.service.cli;
-import org.apache.log4j.Layout;
-import org.apache.log4j.PatternLayout;
-
/**
* CLIServiceUtils.
*
@@ -29,10 +26,6 @@ public class CLIServiceUtils {
private static final char SEARCH_STRING_ESCAPE = '\\';
- public static final Layout verboseLayout = new PatternLayout(
- "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
- public static final Layout nonVerboseLayout = new PatternLayout(
- "%-5p : %m%n");
/**
* Convert a SQL search pattern into an equivalent Java Regex.
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
index 70340bd..fb3921f 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
@@ -6,44 +6,94 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hive.service.cli.operation;
-import java.io.CharArrayWriter;
-import java.util.Enumeration;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
import java.util.regex.Pattern;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
-import org.apache.hive.service.cli.CLIServiceUtils;
-import org.apache.log4j.Appender;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Logger;
-import org.apache.log4j.WriterAppender;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.filter.AbstractFilter;
+import org.apache.logging.log4j.core.layout.PatternLayout;
import com.google.common.base.Joiner;
/**
- * An Appender to divert logs from individual threads to the LogObject they belong to.
+ * Divert appender to redirect operation logs to separate files.
*/
-public class LogDivertAppender extends WriterAppender {
- private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName());
+public class LogDivertAppender
+ extends AbstractOutputStreamAppender<LogDivertAppender.StringOutputStreamManager> {
+ private static final Logger LOG = LogManager.getLogger(LogDivertAppender.class.getName());
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ public static final Layout<? extends Serializable> verboseLayout = PatternLayout.createLayout(
+ "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n", configuration, null, null, true, false, null, null);
+ public static final Layout<? extends Serializable> nonVerboseLayout = PatternLayout.createLayout(
+ "%-5p : %m%n", configuration, null, null, true, false, null, null);
+
private final OperationManager operationManager;
+ private StringOutputStreamManager manager;
private boolean isVerbose;
- private Layout verboseLayout;
+ private final Layout<? extends Serializable> layout;
+
+ /**
+ * Instantiate a WriterAppender and set the output destination to a
+ * new {@link OutputStreamWriter} initialized with <code>os</code>
+ * as its {@link OutputStream}.
+ *
+ * @param name The name of the Appender.
+ * @param filter Filter
+ * @param manager The OutputStreamManager.
+ * @param operationManager Operation manager
+ */
+ protected LogDivertAppender(String name, Filter filter,
+ StringOutputStreamManager manager, OperationManager operationManager,
+ OperationLog.LoggingLevel loggingMode) {
+ super(name, null, filter, false, true, manager);
+ this.operationManager = operationManager;
+ this.manager = manager;
+ this.isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
+ this.layout = getDefaultLayout();
+ }
+
+ public Layout<? extends Serializable> getDefaultLayout() {
+ // There should be a ConsoleAppender. Copy its Layout.
+ Logger root = LogManager.getRootLogger();
+ Layout layout = null;
+
+ for (Appender ap : ((org.apache.logging.log4j.core.Logger) root).getAppenders().values()) {
+ if (ap.getClass().equals(ConsoleAppender.class)) {
+ layout = ap.getLayout();
+ break;
+ }
+ }
+
+ return layout;
+ }
/**
* A log filter that filters messages coming from the logger with the given names.
@@ -52,31 +102,31 @@ public class LogDivertAppender extends WriterAppender {
* they don't generate more logs for themselves when they process logs.
* White list filter is used for less verbose log collection
*/
- private static class NameFilter extends Filter {
+ private static class NameFilter extends AbstractFilter {
private Pattern namePattern;
- private LoggingLevel loggingMode;
+ private OperationLog.LoggingLevel loggingMode;
private OperationManager operationManager;
/* Patterns that are excluded in verbose logging level.
* Filter out messages coming from log processing classes, or we'll run an infinite loop.
*/
private static final Pattern verboseExcludeNamePattern = Pattern.compile(Joiner.on("|").
- join(new String[] {LOG.getName(), OperationLog.class.getName(),
- OperationManager.class.getName()}));
+ join(new String[]{LOG.getName(), OperationLog.class.getName(),
+ OperationManager.class.getName()}));
/* Patterns that are included in execution logging level.
* In execution mode, show only select logger messages.
*/
private static final Pattern executionIncludeNamePattern = Pattern.compile(Joiner.on("|").
- join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter",
- "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
- "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
+ join(new String[]{"org.apache.hadoop.mapreduce.JobSubmitter",
+ "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
+ "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
/* Patterns that are included in performance logging level.
* In performance mode, show execution and performance logger messages.
*/
private static final Pattern performanceIncludeNamePattern = Pattern.compile(
- executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
+ executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
private void setCurrentNamePattern(OperationLog.LoggingLevel mode) {
if (mode == OperationLog.LoggingLevel.VERBOSE) {
@@ -88,26 +138,25 @@ public class LogDivertAppender extends WriterAppender {
}
}
- public NameFilter(
- OperationLog.LoggingLevel loggingMode, OperationManager op) {
+ public NameFilter(OperationLog.LoggingLevel loggingMode, OperationManager op) {
this.operationManager = op;
this.loggingMode = loggingMode;
setCurrentNamePattern(loggingMode);
}
@Override
- public int decide(LoggingEvent ev) {
+ public Result filter(LogEvent event) {
OperationLog log = operationManager.getOperationLogByThread();
boolean excludeMatches = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
if (log == null) {
- return Filter.DENY;
+ return Result.DENY;
}
OperationLog.LoggingLevel currentLoggingMode = log.getOpLoggingLevel();
// If logging is disabled, deny everything.
if (currentLoggingMode == OperationLog.LoggingLevel.NONE) {
- return Filter.DENY;
+ return Result.DENY;
}
// Look at the current session's setting
// and set the pattern and excludeMatches accordingly.
@@ -116,88 +165,58 @@ public class LogDivertAppender extends WriterAppender {
setCurrentNamePattern(loggingMode);
}
- boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches();
+ boolean isMatch = namePattern.matcher(event.getLoggerName()).matches();
if (excludeMatches == isMatch) {
// Deny if this is black-list filter (excludeMatches = true) and it
- // matched
- // or if this is whitelist filter and it didn't match
- return Filter.DENY;
+ // matched or if this is whitelist filter and it didn't match
+ return Result.DENY;
}
- return Filter.NEUTRAL;
+ return Result.NEUTRAL;
}
}
- /** This is where the log message will go to */
- private final CharArrayWriter writer = new CharArrayWriter();
-
- private void setLayout (boolean isVerbose, Layout lo) {
- if (isVerbose) {
- if (lo == null) {
- lo = CLIServiceUtils.verboseLayout;
- LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
- }
- } else {
- lo = CLIServiceUtils.nonVerboseLayout;
- }
- setLayout(lo);
+ public static LogDivertAppender createInstance(OperationManager operationManager,
+ OperationLog.LoggingLevel loggingMode) {
+ return new LogDivertAppender("LogDivertAppender", new NameFilter(loggingMode, operationManager),
+ new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", null),
+ operationManager, loggingMode);
}
- private void initLayout(boolean isVerbose) {
- // There should be a ConsoleAppender. Copy its Layout.
- Logger root = Logger.getRootLogger();
- Layout layout = null;
-
- Enumeration<?> appenders = root.getAllAppenders();
- while (appenders.hasMoreElements()) {
- Appender ap = (Appender) appenders.nextElement();
- if (ap.getClass().equals(ConsoleAppender.class)) {
- layout = ap.getLayout();
- break;
- }
- }
- setLayout(isVerbose, layout);
+ public String getOutput() {
+ return new String(manager.getStream().toByteArray());
}
- public LogDivertAppender(OperationManager operationManager,
- OperationLog.LoggingLevel loggingMode) {
- isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
- initLayout(isVerbose);
- setWriter(writer);
- setName("LogDivertAppender");
- this.operationManager = operationManager;
- this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout;
- addFilter(new NameFilter(loggingMode, operationManager));
+ @Override
+ public void start() {
+ super.start();
}
@Override
- public void doAppend(LoggingEvent event) {
- OperationLog log = operationManager.getOperationLogByThread();
+ public Layout<? extends Serializable> getLayout() {
- // Set current layout depending on the verbose/non-verbose mode.
+ // If there is a logging level change from verbose->non-verbose or vice-versa since
+ // the last subAppend call, change the layout to preserve consistency.
+ OperationLog log = operationManager.getOperationLogByThread();
if (log != null) {
- boolean isCurrModeVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+ isVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+ }
- // If there is a logging level change from verbose->non-verbose or vice-versa since
- // the last subAppend call, change the layout to preserve consistency.
- if (isCurrModeVerbose != isVerbose) {
- isVerbose = isCurrModeVerbose;
- setLayout(isVerbose, verboseLayout);
- }
+ // layout is immutable in log4j2, so we cheat here and return a different layout when
+ // verbosity changes
+ if (isVerbose) {
+ return verboseLayout;
+ } else {
+ return layout == null ? nonVerboseLayout : layout;
}
- super.doAppend(event);
}
- /**
- * Overrides WriterAppender.subAppend(), which does the real logging. No need
- * to worry about concurrency since log4j calls this synchronously.
- */
@Override
- protected void subAppend(LoggingEvent event) {
- super.subAppend(event);
- // That should've gone into our writer. Notify the LogContext.
- String logOutput = writer.toString();
- writer.reset();
+ public void append(LogEvent event) {
+ super.append(event);
+
+ String logOutput = getOutput();
+ manager.reset();
OperationLog log = operationManager.getOperationLogByThread();
if (log == null) {
@@ -206,4 +225,22 @@ public class LogDivertAppender extends WriterAppender {
}
log.writeOperationLog(logOutput);
}
+
+ protected static class StringOutputStreamManager extends OutputStreamManager {
+ ByteArrayOutputStream stream;
+
+ protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+ Layout<?> layout) {
+ super(os, streamName, layout);
+ stream = os;
+ }
+
+ public ByteArrayOutputStream getStream() {
+ return stream;
+ }
+
+ public void reset() {
+ stream.reset();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 9b0a519..304a525 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -41,8 +41,11 @@ import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.RowSetFactory;
import org.apache.hive.service.cli.TableSchema;
import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.log4j.Appender;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
/**
* OperationManager.
@@ -50,7 +53,6 @@ import org.apache.log4j.Logger;
*/
public class OperationManager extends AbstractService {
private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
-
private final Map<OperationHandle, Operation> handleToOperation =
new HashMap<OperationHandle, Operation>();
@@ -83,8 +85,13 @@ public class OperationManager extends AbstractService {
private void initOperationLogCapture(String loggingMode) {
// Register another Appender (with the same layout) that talks to us.
- Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
- Logger.getRootLogger().addAppender(ap);
+ Appender ap = LogDivertAppender.createInstance(this, OperationLog.getLoggingLevel(loggingMode));
+ LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ Configuration configuration = context.getConfiguration();
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(LogManager.getLogger().getName());
+ loggerConfig.addAppender(ap, null, null);
+ context.updateLoggers();
+ ap.start();
}
public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/pom.xml
----------------------------------------------------------------------
diff --git a/shims/common/pom.xml b/shims/common/pom.xml
index 9e9a3b7..dfdec2b 100644
--- a/shims/common/pom.xml
+++ b/shims/common/pom.xml
@@ -41,14 +41,19 @@
<version>${commons-logging.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
deleted file mode 100644
index 224b135..0000000
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.shims;
-
-import org.apache.log4j.Appender;
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Layout;
-import org.apache.log4j.spi.ErrorHandler;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.log4j.spi.OptionHandler;
-
-public class HiveEventCounter implements Appender, OptionHandler {
-
- AppenderSkeleton hadoopEventCounter;
-
- public HiveEventCounter() {
- hadoopEventCounter = ShimLoader.getEventCounter();
- }
-
- @Override
- public void close() {
- hadoopEventCounter.close();
- }
-
- @Override
- public boolean requiresLayout() {
- return hadoopEventCounter.requiresLayout();
- }
-
- @Override
- public void addFilter(Filter filter) {
- hadoopEventCounter.addFilter(filter);
- }
-
- @Override
- public void clearFilters() {
- hadoopEventCounter.clearFilters();
- }
-
- @Override
- public void doAppend(LoggingEvent event) {
- hadoopEventCounter.doAppend(event);
- }
-
- @Override
- public ErrorHandler getErrorHandler() {
- return hadoopEventCounter.getErrorHandler();
- }
-
- @Override
- public Filter getFilter() {
- return hadoopEventCounter.getFilter();
- }
-
- @Override
- public Layout getLayout() {
- return hadoopEventCounter.getLayout();
- }
-
- @Override
- public String getName() {
- return hadoopEventCounter.getName();
- }
-
- @Override
- public void setErrorHandler(ErrorHandler handler) {
- hadoopEventCounter.setErrorHandler(handler);
- }
-
- @Override
- public void setLayout(Layout layout) {
- hadoopEventCounter.setLayout(layout);
- }
-
- @Override
- public void setName(String name) {
- hadoopEventCounter.setName(name);
- }
-
- @Override
- public void activateOptions() {
- hadoopEventCounter.activateOptions();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j.properties b/spark-client/src/test/resources/log4j.properties
deleted file mode 100644
index 93a60cc..0000000
--- a/spark-client/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-log4j.rootCategory=DEBUG, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j2.xml b/spark-client/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..a435069
--- /dev/null
+++ b/spark-client/src/test/resources/log4j2.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="SparkClientLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="spark.log.level">DEBUG</Property>
+ <Property name="spark.root.logger">console</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n"/>
+ </Console>
+ </Appenders>
+
+ <Loggers>
+ <Root level="DEBUG">
+ <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
+ </Root>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 71b51b8..71b79f1 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -32,13 +32,6 @@
</properties>
<dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifectId -->
- <!-- inter-project -->
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
- </dependency>
<!-- test inter-project -->
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index 211678e..2cf7f45 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -64,6 +64,26 @@ limitations under the License.
<version>15.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j.properties b/testutils/ptest2/src/main/resources/log4j.properties
deleted file mode 100644
index edb9696..0000000
--- a/testutils/ptest2/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hive.ptest.logdir=target
-
-log4j.rootLogger=DEBUG,FILE
-log4j.threshhold=ALL
-
-log4j.appender.FILE=org.apache.log4j.RollingFileAppender
-log4j.appender.FILE.File=${hive.ptest.logdir}/ptest.log
-log4j.appender.FILE.MaxFileSize=50MB
-log4j.appender.FILE.MaxBackupIndex=1
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-log4j.logger.org.apache.http=INFO
-log4j.logger.org.springframework=INFO
-log4j.logger.org.jclouds=INFO
-log4j.logger.jclouds=INFO
-log4j.logger.org.apache.hive=DEBUG
-log4j.logger.org.apache.http=TRACE
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN
[34/50] [abbrv] hive git commit: HIVE-8583 Cleanup & Test for
hive.script.operator.env.blacklist (Lars Franke via gates)
Posted by se...@apache.org.
HIVE-8583 Cleanup & Test for hive.script.operator.env.blacklist (Lars Franke via gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d78bb7c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d78bb7c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d78bb7c4
Branch: refs/heads/llap
Commit: d78bb7c49d88a8acf3fd317a06f9563003571f4c
Parents: 87930e2
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Aug 21 10:09:22 2015 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Aug 21 10:09:22 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/ScriptOperator.java | 85 ++++++++++----------
.../hadoop/hive/ql/exec/TestOperators.java | 16 ++++
2 files changed, 60 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d78bb7c4/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
index f2eed44..82ec668 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
@@ -18,25 +18,6 @@
package org.apache.hadoop.hive.ql.exec;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -57,6 +38,26 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.spark.SparkFiles;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
/**
* ScriptOperator.
*
@@ -146,16 +147,19 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
return value;
}
- boolean blackListed(String name) {
+ /**
+ * Checks whether a given configuration name is blacklisted and should not be converted
+ * to an environment variable.
+ */
+ boolean blackListed(Configuration conf, String name) {
if (blackListedConfEntries == null) {
blackListedConfEntries = new HashSet<String>();
- if (hconf != null) {
- String bl = hconf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString());
- if (bl != null && bl.length() > 0) {
+ if (conf != null) {
+ String bl = conf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString(),
+ HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.getDefaultValue());
+ if (bl != null && !bl.isEmpty()) {
String[] bls = bl.split(",");
- for (String b : bls) {
- blackListedConfEntries.add(b);
- }
+ Collections.addAll(blackListedConfEntries, bls);
}
}
}
@@ -171,7 +175,7 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
while (it.hasNext()) {
Map.Entry<String, String> en = it.next();
String name = en.getKey();
- if (!blackListed(name)) {
+ if (!blackListed(conf, name)) {
// String value = (String)en.getValue(); // does not apply variable
// expansion
String value = conf.get(name); // does variable expansion
@@ -306,8 +310,7 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
void displayBrokenPipeInfo() {
if (isLogInfoEnabled) {
LOG.info("The script did not consume all input data. This is considered as an error.");
- LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString()
- + "=true; to ignore it.");
+ LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString() + "=true; to ignore it.");
}
return;
}
@@ -349,12 +352,12 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
}
String[] wrappedCmdArgs = addWrapper(cmdArgs);
- if (isLogInfoEnabled) {
- LOG.info("Executing " + Arrays.asList(wrappedCmdArgs));
- LOG.info("tablename=" + tableName);
- LOG.info("partname=" + partitionName);
- LOG.info("alias=" + alias);
- }
+ if (isLogInfoEnabled) {
+ LOG.info("Executing " + Arrays.asList(wrappedCmdArgs));
+ LOG.info("tablename=" + tableName);
+ LOG.info("partname=" + partitionName);
+ LOG.info("alias=" + alias);
+ }
ProcessBuilder pb = new ProcessBuilder(wrappedCmdArgs);
Map<String, String> env = pb.environment();
@@ -672,9 +675,9 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
long now = System.currentTimeMillis();
// reporter is a member variable of the Operator class.
if (now - lastReportTime > 60 * 1000 && reporter != null) {
- if (isLogInfoEnabled) {
- LOG.info("ErrorStreamProcessor calling reporter.progress()");
- }
+ if (isLogInfoEnabled) {
+ LOG.info("ErrorStreamProcessor calling reporter.progress()");
+ }
lastReportTime = now;
reporter.progress();
}
@@ -730,9 +733,9 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
}
proc.processLine(row);
}
- if (isLogInfoEnabled) {
- LOG.info("StreamThread " + name + " done");
- }
+ if (isLogInfoEnabled) {
+ LOG.info("StreamThread " + name + " done");
+ }
} catch (Throwable th) {
scriptError = th;
http://git-wip-us.apache.org/repos/asf/hive/blob/d78bb7c4/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
index c3a36c0..21dcf86 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -192,6 +193,21 @@ public class TestOperators extends TestCase {
}
}
+ public void testScriptOperatorBlacklistedEnvVarsProcessing() {
+ ScriptOperator scriptOperator = new ScriptOperator();
+
+ Configuration hconf = new JobConf(ScriptOperator.class);
+
+ Map<String, String> env = new HashMap<String, String>();
+
+ HiveConf.setVar(hconf, HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST, "foobar");
+ hconf.set("foobar", "foobar");
+ hconf.set("barfoo", "barfoo");
+ scriptOperator.addJobConfToEnvironment(hconf, env);
+ Assert.assertFalse(env.containsKey("foobar"));
+ Assert.assertTrue(env.containsKey("barfoo"));
+ }
+
public void testScriptOperator() throws Throwable {
try {
System.out.println("Testing Script Operator");
[45/50] [abbrv] hive git commit: HIVE-11450 : Resources are not
cleaned up properly at multiple places (Nezih Yigitbasi via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11450 : Resources are not cleaned up properly at multiple places (Nezih Yigitbasi via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2a1764f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2a1764f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2a1764f
Branch: refs/heads/llap
Commit: e2a1764f07d6fa242a29c25c6cc3d1a13405d038
Parents: 9d3d3d0
Author: Nezih Yigitbasi <ny...@netflix.com>
Authored: Sun Aug 23 22:40:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 24 10:20:10 2015 -0700
----------------------------------------------------------------------
beeline/src/java/org/apache/hive/beeline/BeeLine.java | 13 ++++++-------
.../src/java/org/apache/hive/beeline/BeeLineOpts.java | 12 ++++++------
.../java/org/apache/hive/beeline/HiveSchemaTool.java | 14 ++++++++++----
.../src/java/org/apache/hive/jdbc/HiveConnection.java | 14 ++++++--------
4 files changed, 28 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a1764f/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 1519619..3cd2a8b 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -844,7 +844,7 @@ public class BeeLine implements Closeable {
consoleReader.setExpandEvents(false);
// setup history
- ByteArrayOutputStream hist = null;
+ ByteArrayOutputStream hist = new ByteArrayOutputStream();
if (new File(getOpts().getHistoryFile()).isFile()) {
try {
// save the current contents of the history buffer. This gets
@@ -852,13 +852,12 @@ public class BeeLine implements Closeable {
// input will clobber the history input, but setting the
// input before the output will cause the previous commands
// to not be saved to the buffer.
- FileInputStream historyIn = new FileInputStream(getOpts().getHistoryFile());
- hist = new ByteArrayOutputStream();
- int n;
- while ((n = historyIn.read()) != -1) {
- hist.write(n);
+ try (FileInputStream historyIn = new FileInputStream(getOpts().getHistoryFile())) {
+ int n;
+ while ((n = historyIn.read()) != -1) {
+ hist.write(n);
+ }
}
- historyIn.close();
} catch (Exception e) {
handleException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a1764f/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index a31c49c..3388391 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -154,9 +154,9 @@ class BeeLineOpts implements Completer {
public void save() throws IOException {
- OutputStream out = new FileOutputStream(rcFile);
- save(out);
- out.close();
+ try (OutputStream out = new FileOutputStream(rcFile)) {
+ save(out);
+ }
}
public void save(OutputStream out) throws IOException {
@@ -208,9 +208,9 @@ class BeeLineOpts implements Completer {
public void load() throws IOException {
- InputStream in = new FileInputStream(rcFile);
- load(in);
- in.close();
+ try (InputStream in = new FileInputStream(rcFile)) {
+ load(in);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a1764f/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index 2477e5f..d5d635a 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -140,18 +140,24 @@ public class HiveSchemaTool {
} else {
versionQuery = "select t.SCHEMA_VERSION from VERSION t";
}
- try {
- Statement stmt = metastoreConn.createStatement();
- ResultSet res = stmt.executeQuery(versionQuery);
+ try(Statement stmt = metastoreConn.createStatement();
+ ResultSet res = stmt.executeQuery(versionQuery)) {
if (!res.next()) {
throw new HiveMetaException("Didn't find version data in metastore");
}
String currentSchemaVersion = res.getString(1);
- metastoreConn.close();
return currentSchemaVersion;
} catch (SQLException e) {
throw new HiveMetaException("Failed to get schema version.", e);
}
+ finally {
+ try {
+ metastoreConn.close();
+ } catch (SQLException e) {
+ System.err.println("Failed to close the metastore connection");
+ e.printStackTrace(System.err);
+ }
+ }
}
// test the connection metastore using the config property
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a1764f/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 3b7e004..a9dac03 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -959,15 +959,13 @@ public class HiveConnection implements java.sql.Connection {
if (isClosed) {
throw new SQLException("Connection is closed");
}
- Statement stmt = createStatement();
- ResultSet res = stmt.executeQuery("SELECT current_database()");
- if (!res.next()) {
- throw new SQLException("Failed to get schema information");
+ try (Statement stmt = createStatement();
+ ResultSet res = stmt.executeQuery("SELECT current_database()")) {
+ if (!res.next()) {
+ throw new SQLException("Failed to get schema information");
+ }
+ return res.getString(1);
}
- String schemaName = res.getString(1);
- res.close();
- stmt.close();
- return schemaName;
}
/*
[08/50] [abbrv] hive git commit: HIVE-11304: Migrate to Log4j2 from
Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..992462e
--- /dev/null
+++ b/testutils/ptest2/src/main/resources/log4j2.xml
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="PTest2Log4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.ptest.log.threshold">ALL</Property>
+ <Property name="hive.ptest.log.level">DEBUG</Property>
+ <Property name="hive.ptest.root.logger">FILE</Property>
+ <Property name="hive.ptest.log.dir">target</Property>
+ <Property name="hive.ptest.log.file">ptest.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Size based Rolling File Appender -->
+ <RollingFile name="FILE" fileName="${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}"
+ filePattern="${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}.%i">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="50 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="1"/>
+ </RollingFile>
+
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.ptest.log.threshold}">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}" level="${sys:hive.ptest.log.level}"/>
+ </Root>
+
+ <Logger name="org.apache.http" level="INFO">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="org.springframework" level="INFO">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="org.jclouds" level="INFO">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="jclouds" level="INFO">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.hive" level="DEBUG">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.http" level="TRACE">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+
+ <!-- Silence useless ZK logs -->
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
+ </Logger>
+
+</Configuration>
[41/50] [abbrv] hive git commit: HIVE-11506 : Casting varchar/char
type to string cannot be vectorized (Navis via Ashutosh Chauhan)
Posted by se...@apache.org.
HIVE-11506 : Casting varchar/char type to string cannot be vectorized (Navis via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2de6fda0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2de6fda0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2de6fda0
Branch: refs/heads/llap
Commit: 2de6fda0f81072a6ad1af103f2ddec49b73f4eab
Parents: e2d148b
Author: Navis Ryu <na...@apache.org>
Authored: Mon Aug 10 17:36:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sun Aug 23 21:32:00 2015 -0700
----------------------------------------------------------------------
.../ql/exec/vector/VectorizationContext.java | 7 +-
.../expressions/CastStringGroupToString.java | 40 ++++++++
.../queries/clientpositive/vectorized_casts.q | 6 ++
.../tez/vector_char_mapjoin1.q.out | 1 +
.../tez/vector_varchar_mapjoin1.q.out | 1 +
.../clientpositive/tez/vectorized_casts.q.out | 99 +++++++++++++-------
.../clientpositive/vector_char_mapjoin1.q.out | 1 +
.../vector_varchar_mapjoin1.q.out | 1 +
.../clientpositive/vectorized_casts.q.out | 66 +++++++------
9 files changed, 155 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 5b702bb..34f9329 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode;
import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
-import org.apache.hadoop.hive.ql.exec.vector.AggregateDefinition;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount;
@@ -1023,9 +1022,7 @@ public class VectorizationContext {
VectorExpressionDescriptor.Descriptor descriptor = builder.build();
Class<?> vclass = this.vMap.getVectorExpressionClass(udfClass, descriptor);
if (vclass == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No vector udf found for "+udfClass.getSimpleName() + ", descriptor: "+descriptor);
- }
+ LOG.info("No vector udf found for " + udfClass.getSimpleName() + ", descriptor: " + descriptor);
return null;
}
Mode childrenMode = getChildrenMode(mode, udfClass);
@@ -1536,6 +1533,8 @@ public class VectorizationContext {
return createVectorExpression(CastDecimalToString.class, childExpr, Mode.PROJECTION, returnType);
} else if (isDateFamily(inputType)) {
return createVectorExpression(CastDateToString.class, childExpr, Mode.PROJECTION, returnType);
+ } else if (isStringFamily(inputType)) {
+ return createVectorExpression(CastStringGroupToString.class, childExpr, Mode.PROJECTION, returnType);
}
/* The string type is deliberately omitted -- the planner removes string to string casts.
* Timestamp, float, and double types are handled by the legacy code path. See isLegacyPathUDF.
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
new file mode 100644
index 0000000..bbc770c
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+
+// cast string group to string (varchar to string, etc.)
+public class CastStringGroupToString extends StringUnaryUDFDirect {
+
+ private static final long serialVersionUID = 1L;
+
+ public CastStringGroupToString() {
+ super();
+ }
+
+ public CastStringGroupToString(int inputColumn, int outputColumn) {
+ super(inputColumn, outputColumn);
+ }
+
+ @Override
+ protected void func(BytesColumnVector outV, byte[][] vector, int[] start, int[] length, int i) {
+ outV.setVal(i, vector[i], start[i], length[i]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/queries/clientpositive/vectorized_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_casts.q b/ql/src/test/queries/clientpositive/vectorized_casts.q
index ade3645..8e0bdbc 100644
--- a/ql/src/test/queries/clientpositive/vectorized_casts.q
+++ b/ql/src/test/queries/clientpositive/vectorized_casts.q
@@ -1,8 +1,10 @@
set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
SET hive.vectorized.execution.enabled = true;
-- SORT_QUERY_RESULTS
+-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
-- Test type casting in vectorized mode to verify end-to-end functionality.
explain
@@ -68,6 +70,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -140,6 +144,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
index f495f95..cdb24cc 100644
--- a/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
@@ -366,6 +366,7 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 2 Data size: 215 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string)
+ Execution mode: vectorized
Map 3
Map Operator Tree:
TableScan
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out b/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
index ae5be44..b453b0e 100644
--- a/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
@@ -354,6 +354,7 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: string)
+ Execution mode: vectorized
Map 3
Map Operator Tree:
TableScan
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
index 1dd07b1..19c41ed 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
@@ -1,5 +1,6 @@
PREHOOK: query: -- SORT_QUERY_RESULTS
+-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
-- Test type casting in vectorized mode to verify end-to-end functionality.
explain
@@ -65,6 +66,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -76,6 +79,7 @@ where cbigint % 250 = 0
PREHOOK: type: QUERY
POSTHOOK: query: -- SORT_QUERY_RESULTS
+-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
-- Test type casting in vectorized mode to verify end-to-end functionality.
explain
@@ -141,6 +145,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -151,21 +157,40 @@ from alltypesorc
where cbigint % 250 = 0
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-0 is a root stage
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: alltypesorc
+ Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((cbigint % 250) = 0) (type: boolean)
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double),
UDFToDouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type:
string), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToString(CAST( cstring1 AS CHAR(10)) (type: string), UDFToString(CAST( cstring1 AS varchar(10))) (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59, _col60, _col61
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
- TableScan
- alias: alltypesorc
- Filter Operator
- predicate: ((cbigint % 250) = 0) (type: boolean)
- Select Operator
- expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToDou
ble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string),
UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59
- ListSink
+ ListSink
PREHOOK: query: select
-- to boolean
@@ -229,6 +254,8 @@ PREHOOK: query: select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -302,6 +329,8 @@ POSTHOOK: query: select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -313,29 +342,29 @@ where cbigint % 250 = 0
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.7360096E8
-true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb 8.0 -2.05872868E8 0.9893582466233818 -1.02936432E8
-true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 8.0 -1.323242276E9 0.9893582466233818 -6.6162112E8
-true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 8.0 -1.339264622E9 0.9893582466233818 -6.6963232E8
-true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179648E8
-true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553857E8
-true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 11.0 -6.70900834E8 -0.9999902065507035 -3.35450431E8
-true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 11.0 -1.29231964E8 -0.9999902065507035 -6.4615983E7
-true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988993E8
-true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867265E8
-true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL -14.0 NULL -0.9906073556948704 NULL
-true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -5 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL -21.0 NULL -0.8366556385360561 NULL
-true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -7 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL -24.0 NULL 0.9055783620066238 NULL
-true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL -30.0 NULL 0.9880316240928618 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -15 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -7 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -2 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.3860000000000001 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL -38.0 NULL -0.2963685787093853 NULL
-true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL -5.0 NULL 0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -6 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL -50.0 NULL 0.26237485370392877 NULL
-true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL -59.0 NULL -0.6367380071391379 NULL
-true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -8 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL -60.0 NULL 0.3048106211022167 NULL
-true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL -8.0 NULL -0.9893582466233818 NULL
-true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -15 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL 20.0 NULL 0.9129452507276277 NULL
-true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -10 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL 48.0 NULL -0.7682546613236668 NULL
-true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL 5.0 NULL -0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL 59.0 NULL 0.6367380071391379 NULL
+true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O 2yK4Bx76O 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
+true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb eJROSNhugc eJROSNhugc 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
+true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 L15l8i5k55 L15l8i5k55 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
+true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 3r3sDvfUkG 3r3sDvfUkG 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
+true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q e005B5q e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
+true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r q8M86Fx0r q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
+true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 dOYnqgaXoJ dOYnqgaXoJ 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
+true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 8J5OB7K26P 8J5OB7K26P 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
+true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
+true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
+true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
+true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
+true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
+true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
+true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
+true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
+true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
+true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
+true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
+true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
+true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
+true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
+true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
index 744bfb3..af5f0de 100644
--- a/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
@@ -371,6 +371,7 @@ STAGE PLANS:
value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string)
Local Work:
Map Reduce Local Work
+ Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out b/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
index 2994bff..936ef78 100644
--- a/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
@@ -359,6 +359,7 @@ STAGE PLANS:
value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: string)
Local Work:
Map Reduce Local Work
+ Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vectorized_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_casts.q.out b/ql/src/test/results/clientpositive/vectorized_casts.q.out
index 1113453..3c28064 100644
--- a/ql/src/test/results/clientpositive/vectorized_casts.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_casts.q.out
@@ -1,5 +1,6 @@
PREHOOK: query: -- SORT_QUERY_RESULTS
+-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
-- Test type casting in vectorized mode to verify end-to-end functionality.
explain
@@ -65,6 +66,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -76,6 +79,7 @@ where cbigint % 250 = 0
PREHOOK: type: QUERY
POSTHOOK: query: -- SORT_QUERY_RESULTS
+-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
-- Test type casting in vectorized mode to verify end-to-end functionality.
explain
@@ -141,6 +145,8 @@ select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -165,8 +171,8 @@ STAGE PLANS:
predicate: ((cbigint % 250) = 0) (type: boolean)
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToD
ouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string
), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59
+ expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToD
ouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string
), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToString(CAST( cstring1 AS CHAR(10)) (type: string), UDFToString(CAST( cstring1 AS varchar(10))) (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59, _col60, _col61
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
@@ -245,6 +251,8 @@ PREHOOK: query: select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -318,6 +326,8 @@ POSTHOOK: query: select
,cast (cbigint * 0 as string)
,cast (ctimestamp1 as string)
,cast (cstring1 as string)
+ ,cast (cast (cstring1 as char(10)) as string)
+ ,cast (cast (cstring1 as varchar(10)) as string)
-- nested and expression arguments
,cast (cast (cfloat as int) as float)
,cast (cint * 2 as double)
@@ -329,29 +339,29 @@ where cbigint % 250 = 0
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
-true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
-true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
-true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
-true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
-true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
-true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
-true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
-true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
-true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
-true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL -14.0 NULL -0.9906073556948704 NULL
-true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL -21.0 NULL -0.8366556385360561 NULL
-true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL -24.0 NULL 0.9055783620066238 NULL
-true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL -30.0 NULL 0.9880316240928618 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL -36.0 NULL 0.9917788534431158 NULL
-true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL -38.0 NULL -0.2963685787093853 NULL
-true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL -5.0 NULL 0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL -50.0 NULL 0.26237485370392877 NULL
-true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL -59.0 NULL -0.6367380071391379 NULL
-true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL -60.0 NULL 0.3048106211022167 NULL
-true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL -8.0 NULL -0.9893582466233818 NULL
-true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL 20.0 NULL 0.9129452507276277 NULL
-true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL 48.0 NULL -0.7682546613236668 NULL
-true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL 5.0 NULL -0.9589242746631385 NULL
-true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL 59.0 NULL 0.6367380071391379 NULL
+true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O 2yK4Bx76O 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
+true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb eJROSNhugc eJROSNhugc 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
+true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 L15l8i5k55 L15l8i5k55 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
+true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 3r3sDvfUkG 3r3sDvfUkG 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
+true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q e005B5q e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
+true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r q8M86Fx0r q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
+true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 dOYnqgaXoJ dOYnqgaXoJ 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
+true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 8J5OB7K26P 8J5OB7K26P 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
+true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
+true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
+true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
+true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
+true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
+true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
+true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
+true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
+true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
+true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
+true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
+true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
+true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
+true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
+true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
+true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL
[32/50] [abbrv] hive git commit: HIVE-11602: Support Struct with
different field types in query (Jesus Camacho Rodriguez,
reviewed by Hari Sankar Sivarama Subramaniyan)
Posted by se...@apache.org.
HIVE-11602: Support Struct with different field types in query (Jesus Camacho Rodriguez, reviewed by Hari Sankar Sivarama Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0012864b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0012864b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0012864b
Branch: refs/heads/llap
Commit: 0012864b8d370f948ac2cb875e4264100390d1d4
Parents: ab03dc9
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed Aug 19 12:52:16 2015 +0300
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Aug 20 15:17:40 2015 +0300
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 61 ++++++++++++++++++--
.../ql/udf/generic/GenericUDFBaseNumeric.java | 4 +-
ql/src/test/queries/clientpositive/structin.q | 6 ++
.../test/results/clientpositive/structin.q.out | 44 ++++++++++++++
4 files changed, 109 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 9edcc4d..4c1c53e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -132,6 +132,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -765,10 +766,11 @@ public final class FunctionRegistry {
return null;
}
- public static PrimitiveCategory getCommonCategory(TypeInfo a, TypeInfo b) {
+ public static PrimitiveCategory getPrimitiveCommonCategory(TypeInfo a, TypeInfo b) {
if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) {
return null;
}
+
PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory();
PrimitiveCategory pcB = ((PrimitiveTypeInfo)b).getPrimitiveCategory();
@@ -802,10 +804,61 @@ public final class FunctionRegistry {
return a;
}
- PrimitiveCategory commonCat = getCommonCategory(a, b);
- if (commonCat == null)
+ // We try to infer a common primitive category
+ PrimitiveCategory commonCat = getPrimitiveCommonCategory(a, b);
+ if (commonCat != null) {
+ return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, commonCat);
+ }
+ // It is not primitive; check if it is a struct and we can infer a common class
+ if (a.getCategory() == Category.STRUCT && b.getCategory() == Category.STRUCT) {
+ return getCommonClassForStruct((StructTypeInfo)a, (StructTypeInfo)b);
+ }
+ return null;
+ }
+
+ /**
+ * Find a common class that objects of both StructTypeInfo a and StructTypeInfo b can
+ * convert to. This is used for places other than comparison.
+ *
+ * @return null if no common class could be found.
+ */
+ public static TypeInfo getCommonClassForStruct(StructTypeInfo a, StructTypeInfo b) {
+ if (a == b || a.equals(b)) {
+ return a;
+ }
+
+ List<String> names = new ArrayList<String>();
+ List<TypeInfo> typeInfos = new ArrayList<TypeInfo>();
+
+ Iterator<String> namesIterator = a.getAllStructFieldNames().iterator();
+ Iterator<String> otherNamesIterator = b.getAllStructFieldNames().iterator();
+
+ // Compare the field names using ignore-case semantics
+ while (namesIterator.hasNext() && otherNamesIterator.hasNext()) {
+ String name = namesIterator.next();
+ if (!name.equalsIgnoreCase(otherNamesIterator.next())) {
+ return null;
+ }
+ names.add(name);
+ }
+
+ // Different number of field names
+ if (namesIterator.hasNext() || otherNamesIterator.hasNext()) {
return null;
- return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, commonCat);
+ }
+
+ // Compare the field types
+ ArrayList<TypeInfo> fromTypes = a.getAllStructFieldTypeInfos();
+ ArrayList<TypeInfo> toTypes = b.getAllStructFieldTypeInfos();
+ for (int i = 0; i < fromTypes.size(); i++) {
+ TypeInfo commonType = getCommonClass(fromTypes.get(i), toTypes.get(i));
+ if (commonType == null) {
+ return null;
+ }
+ typeInfos.add(commonType);
+ }
+
+ return TypeInfoFactory.getStructTypeInfo(names, typeInfos);
}
public static boolean implicitConvertible(PrimitiveCategory from, PrimitiveCategory to) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
index ca5c459..ef6ef11 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
@@ -245,7 +245,7 @@ public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {
}
// Use type promotion
- PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
+ PrimitiveCategory commonCat = FunctionRegistry.getPrimitiveCommonCategory(left, right);
if (commonCat == PrimitiveCategory.DECIMAL) {
// Hive 0.12 behavior where double * decimal -> decimal is gone.
return TypeInfoFactory.doubleTypeInfo;
@@ -267,7 +267,7 @@ public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {
PrimitiveTypeInfo right = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(rightOI);
// Now we are handling exact types. Base implementation handles type promotion.
- PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
+ PrimitiveCategory commonCat = FunctionRegistry.getPrimitiveCommonCategory(left, right);
if (commonCat == PrimitiveCategory.DECIMAL) {
return deriveResultDecimalTypeInfo();
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/test/queries/clientpositive/structin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/structin.q b/ql/src/test/queries/clientpositive/structin.q
index 48b31f3..35498bf 100644
--- a/ql/src/test/queries/clientpositive/structin.q
+++ b/ql/src/test/queries/clientpositive/structin.q
@@ -15,3 +15,9 @@ struct('1234-1111-0074019112','1'),
struct('1234-1111-0074019610','1'),
struct('1234-1111-0074022106','1')
);
+
+explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695',1)
+);
http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/test/results/clientpositive/structin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/structin.q.out b/ql/src/test/results/clientpositive/structin.q.out
index 81c792a..f176e79 100644
--- a/ql/src/test/results/clientpositive/structin.q.out
+++ b/ql/src/test/results/clientpositive/structin.q.out
@@ -64,3 +64,47 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695',1)
+)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
+IN (
+struct('1234-1111-0074578664','3'),
+struct('1234-1111-0074578695',1)
+)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t11
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Filter Operator
+ predicate: (struct(id,lineid)) IN (const struct('1234-1111-0074578664','3'), const struct('1234-1111-0074578695',1)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: id (type: string), lineid (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+