You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2016/12/24 17:16:51 UTC
hive git commit: HIVE-15360: Nested column pruning: add pruned column
paths to explain output (Chao Sun, reviewed by Ferdinand Xu)
Repository: hive
Updated Branches:
refs/heads/master cc9012b42 -> ab0f9cab3
HIVE-15360: Nested column pruning: add pruned column paths to explain output (Chao Sun, reviewed by Ferdinand Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab0f9cab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab0f9cab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab0f9cab
Branch: refs/heads/master
Commit: ab0f9cab38b17bfcbe248e323b8340f72d5760db
Parents: cc9012b
Author: Chao Sun <su...@apache.org>
Authored: Wed Dec 21 14:20:58 2016 -0800
Committer: Chao Sun <su...@apache.org>
Committed: Sat Dec 24 09:16:31 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/plan/TableScanDesc.java | 11 +++++++++++
.../test/results/clientpositive/case_sensitivity.q.out | 1 +
.../test/results/clientpositive/input_testxpath.q.out | 1 +
.../test/results/clientpositive/input_testxpath3.q.out | 1 +
.../results/clientpositive/nested_column_pruning.q.out | 13 +++++++++++++
ql/src/test/results/clientpositive/udf_coalesce.q.out | 1 +
6 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 2cb0935..b434cfa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -218,6 +218,17 @@ public class TableScanDesc extends AbstractOperatorDesc {
return neededColumns;
}
+ @Explain(displayName = "Pruned Column Paths")
+ public List<String> getPrunedColumnPaths() {
+ List<String> result = new ArrayList<>();
+ for (String p : neededNestedColumnPaths) {
+ if (p.indexOf('.') >= 0) {
+ result.add(p);
+ }
+ }
+ return result;
+ }
+
public void setReferencedColumns(List<String> referencedColumns) {
this.referencedColumns = referencedColumns;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/test/results/clientpositive/case_sensitivity.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/case_sensitivity.q.out b/ql/src/test/results/clientpositive/case_sensitivity.q.out
index b3969cc..1952c8a 100644
--- a/ql/src/test/results/clientpositive/case_sensitivity.q.out
+++ b/ql/src/test/results/clientpositive/case_sensitivity.q.out
@@ -30,6 +30,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src_thrift
+ Pruned Column Paths: lintstring.mystring
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (lint[0] > 0) (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/test/results/clientpositive/input_testxpath.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_testxpath.q.out b/ql/src/test/results/clientpositive/input_testxpath.q.out
index e07628a..422a936 100644
--- a/ql/src/test/results/clientpositive/input_testxpath.q.out
+++ b/ql/src/test/results/clientpositive/input_testxpath.q.out
@@ -30,6 +30,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src_thrift
+ Pruned Column Paths: lintstring.mystring
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: lint[1] (type: int), lintstring[0].mystring (type: string), mstringstring['key_2'] (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/test/results/clientpositive/input_testxpath3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_testxpath3.q.out b/ql/src/test/results/clientpositive/input_testxpath3.q.out
index 5779bfd..2fe5c61 100644
--- a/ql/src/test/results/clientpositive/input_testxpath3.q.out
+++ b/ql/src/test/results/clientpositive/input_testxpath3.q.out
@@ -16,6 +16,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src_thrift
+ Pruned Column Paths: lintstring.myint
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: mstringstring['key_9'] (type: string), lintstring.myint (type: array<int>)
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/test/results/clientpositive/nested_column_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nested_column_pruning.q.out b/ql/src/test/results/clientpositive/nested_column_pruning.q.out
index f01e3ea..c501c6a 100644
--- a/ql/src/test/results/clientpositive/nested_column_pruning.q.out
+++ b/ql/src/test/results/clientpositive/nested_column_pruning.q.out
@@ -153,6 +153,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f1
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f1 (type: boolean)
@@ -195,6 +196,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f1, s1.f2
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f1 (type: boolean), s1.f2 (type: string)
@@ -239,6 +241,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int)
@@ -283,6 +286,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3.f5
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3.f5 (type: double)
@@ -327,6 +331,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3.f4, s2.f8.f9
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3.f4 (type: int), s2.f8.f9 (type: boolean)
@@ -373,6 +378,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f2, s1.f1
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (s1.f1 = false) (type: boolean)
@@ -418,6 +424,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3.f5, s1.f3.f4
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (s1.f3.f4 = 4) (type: boolean)
@@ -463,6 +470,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f2, s2.f8
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((s1.f2 = 'foo') and (size(s2.f8.f10) > 1) and s2.f8.f11['key1']) (type: boolean)
@@ -516,6 +524,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s2.f8.f10
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Lateral View Forward
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
@@ -657,6 +666,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s2.f8.f10, s1.f3.f4
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (s2.f8.f10[1] pmod s1.f3.f4) (type: int)
@@ -703,6 +713,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3.f5, s1.f3.f4
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3.f5 (type: double), s1.f3.f4 (type: int)
@@ -764,6 +775,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int)
@@ -826,6 +838,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: nested_tbl_1
+ Pruned Column Paths: s1.f3
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: s1.f3 (type: struct<f4:int,f5:double>), s1.f3.f4 (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/ab0f9cab/ql/src/test/results/clientpositive/udf_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_coalesce.q.out b/ql/src/test/results/clientpositive/udf_coalesce.q.out
index 361d173..dc3d9df 100644
--- a/ql/src/test/results/clientpositive/udf_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/udf_coalesce.q.out
@@ -140,6 +140,7 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: src_thrift
+ Pruned Column Paths: lintstring.mystring
Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: COALESCE(lint[1],999) (type: int), COALESCE(lintstring[0].mystring,'999') (type: string), COALESCE(mstringstring['key_2'],'999') (type: string)