You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2017/12/12 18:55:26 UTC
hive git commit: HIVE-18191: Vectorization: Add validation of
TableScanOperator (gather statistics) back (Matt McCline,
reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master d6ce23d53 -> 1320d2b31
HIVE-18191: Vectorization: Add validation of TableScanOperator (gather statistics) back (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1320d2b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1320d2b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1320d2b3
Branch: refs/heads/master
Commit: 1320d2b31a358f319d8eeb4aa4f781d47bb3f4b4
Parents: d6ce23d
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Dec 12 12:55:11 2017 -0600
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Dec 12 12:55:11 2017 -0600
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/TableScanOperator.java | 10 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 7 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 4 +
.../clientpositive/vector_gather_stats.q | 26 +++++
.../clientpositive/tez/explainuser_3.q.out | 2 +-
.../clientpositive/vector_gather_stats.q.out | 101 +++++++++++++++++++
6 files changed, 143 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
index c76026b..c0138f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
@@ -115,11 +115,7 @@ public class TableScanOperator extends Operator<TableScanDesc> implements
@Override
public void process(Object row, int tag) throws HiveException {
if (rowLimit >= 0) {
- if (row instanceof VectorizedRowBatch) {
- // We need to check with 'instanceof' instead of just checking
- // vectorized because the row can be a VectorizedRowBatch when
- // FetchOptimizer kicks in even if the operator pipeline is not
- // vectorized
+ if (vectorized) {
VectorizedRowBatch batch = (VectorizedRowBatch) row;
if (currCount >= rowLimit) {
setDone(true);
@@ -266,6 +262,10 @@ public class TableScanOperator extends Operator<TableScanDesc> implements
currentStat = null;
stats = new HashMap<String, Stat>();
+ /*
+ * This TableScanDesc flag is strictly set by the Vectorizer class for vectorized MapWork
+ * vertices.
+ */
vectorized = conf.isVectorized();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 6500682..8ce2c33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1513,7 +1513,7 @@ public class Vectorizer implements PhysicalPlanResolver {
LOG.info("Examining input format to see if vectorization is enabled.");
ImmutablePair<String,TableScanOperator> onlyOneTableScanPair = verifyOnlyOneTableScanOperator(mapWork);
- if (onlyOneTableScanPair == null) {
+ if (onlyOneTableScanPair == null) {
VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason();
Preconditions.checkState(notVectorizedReason != null);
mapWork.setVectorizationEnabledConditionsNotMet(Arrays.asList(new String[] {notVectorizedReason.toString()}));
@@ -1638,6 +1638,11 @@ public class Vectorizer implements PhysicalPlanResolver {
// Set "global" member indicating where to store "not vectorized" information if necessary.
currentBaseWork = mapWork;
+ if (!validateTableScanOperator(tableScanOperator, mapWork)) {
+
+ // The "not vectorized" information has been stored in the MapWork vertex.
+ return false;
+ }
try {
validateAndVectorizeMapOperators(tableScanOperator, isTezOrSpark, vectorTaskColumnInfo);
} catch (VectorizerCannotVectorizeException e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 4b7d2b4..8d966c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -495,6 +495,10 @@ public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherD
return new TableScanOperatorExplainVectorization(this, vectorTableScanDesc);
}
+ /*
+ * This TableScanDesc flag is strictly set by the Vectorizer class for vectorized MapWork
+ * vertices.
+ */
public void setVectorized(boolean vectorized) {
this.vectorized = vectorized;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/test/queries/clientpositive/vector_gather_stats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_gather_stats.q b/ql/src/test/queries/clientpositive/vector_gather_stats.q
new file mode 100644
index 0000000..deaca0e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_gather_stats.q
@@ -0,0 +1,26 @@
+set hive.vectorized.execution.enabled=true;
+
+-- HIVE-18191
+
+create table cd
+(
+ cd_demo_sk int,
+ cd_gender string,
+ cd_marital_status string,
+ cd_purchase_estimate int,
+ cd_credit_rating string,
+ cd_dep_count int,
+ cd_dep_employed_count int,
+ cd_dep_college_count int
+)
+partitioned by
+(
+ cd_education_status string
+);
+alter table cd add partition (cd_education_status='Primary');
+insert into table cd partition (cd_education_status='Primary') values (1, 'M', 'M', 500, 'Good', 0, 0, 0);
+
+explain vectorization detail
+analyze table cd partition (cd_education_status) compute statistics;
+
+analyze table cd partition (cd_education_status) compute statistics;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index 8b1ee28..b299377 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -191,7 +191,7 @@ POSTHOOK: type: QUERY
Stage-2
Stats Work{}
Stage-0
- Map 1 vectorized
+ Map 1
TableScan [TS_0] (rows=500 width=10)
default@src,src,Tbl:COMPLETE,Col:COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/1320d2b3/ql/src/test/results/clientpositive/vector_gather_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_gather_stats.q.out b/ql/src/test/results/clientpositive/vector_gather_stats.q.out
new file mode 100644
index 0000000..cf9bc85
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_gather_stats.q.out
@@ -0,0 +1,101 @@
+PREHOOK: query: create table cd
+(
+ cd_demo_sk int,
+ cd_gender string,
+ cd_marital_status string,
+ cd_purchase_estimate int,
+ cd_credit_rating string,
+ cd_dep_count int,
+ cd_dep_employed_count int,
+ cd_dep_college_count int
+)
+partitioned by
+(
+ cd_education_status string
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@cd
+POSTHOOK: query: create table cd
+(
+ cd_demo_sk int,
+ cd_gender string,
+ cd_marital_status string,
+ cd_purchase_estimate int,
+ cd_credit_rating string,
+ cd_dep_count int,
+ cd_dep_employed_count int,
+ cd_dep_college_count int
+)
+partitioned by
+(
+ cd_education_status string
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@cd
+PREHOOK: query: alter table cd add partition (cd_education_status='Primary')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@cd
+POSTHOOK: query: alter table cd add partition (cd_education_status='Primary')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@cd
+POSTHOOK: Output: default@cd@cd_education_status=Primary
+PREHOOK: query: insert into table cd partition (cd_education_status='Primary') values (1, 'M', 'M', 500, 'Good', 0, 0, 0)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@cd@cd_education_status=Primary
+POSTHOOK: query: insert into table cd partition (cd_education_status='Primary') values (1, 'M', 'M', 500, 'Good', 0, 0, 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@cd@cd_education_status=Primary
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_credit_rating SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_demo_sk EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_dep_college_count EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_dep_count EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_dep_employed_count EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_gender SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_marital_status SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: cd PARTITION(cd_education_status=Primary).cd_purchase_estimate EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
+PREHOOK: query: explain vectorization detail
+analyze table cd partition (cd_education_status) compute statistics
+PREHOOK: type: QUERY
+POSTHOOK: query: explain vectorization detail
+analyze table cd partition (cd_education_status) compute statistics
+POSTHOOK: type: QUERY
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: cd
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: NONE
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ inputFileFormats: org.apache.hadoop.mapred.TextInputFormat
+ notVectorizedReason: TABLESCAN operator: gather stats not supported
+ vectorized: false
+
+ Stage: Stage-1
+ Stats Work
+ Basic Stats Work:
+
+PREHOOK: query: analyze table cd partition (cd_education_status) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@cd
+PREHOOK: Input: default@cd@cd_education_status=Primary
+PREHOOK: Output: default@cd
+PREHOOK: Output: default@cd@cd_education_status=Primary
+POSTHOOK: query: analyze table cd partition (cd_education_status) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@cd
+POSTHOOK: Input: default@cd@cd_education_status=Primary
+POSTHOOK: Output: default@cd
+POSTHOOK: Output: default@cd@cd_education_status=Primary