You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/11/25 16:44:24 UTC
svn commit: r1641640 - in /hive/branches/spark/ql/src:
java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
test/results/clientpositive/spark/limit_partition_metadataonly.q.out
Author: xuefu
Date: Tue Nov 25 15:44:24 2014
New Revision: 1641640
URL: http://svn.apache.org/r1641640
Log:
HIVE-8788: UT: fix partition test case [Spark Branch] (Chinna via Xuefu)
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java?rev=1641640&r1=1641639&r2=1641640&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java Tue Nov 25 15:44:24 2014
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.ql.plan.Ba
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceWork;
import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.UnionWork;
@@ -214,6 +216,20 @@ public class SparkTask extends Task<Spar
return result;
}
+ @Override
+ public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) {
+ List<BaseWork> children = getWork().getChildren(mapWork);
+ if (children.size() != 1) {
+ return null;
+ }
+
+ if (!(children.get(0) instanceof ReduceWork)) {
+ return null;
+ }
+
+ return ((ReduceWork) children.get(0)).getReducer();
+ }
+
public SparkCounters getSparkCounters() {
return sparkCounters;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1641640&r1=1641639&r2=1641640&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java Tue Nov 25 15:44:24 2014
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.ql.lib.Rul
import org.apache.hadoop.hive.ql.lib.TypeRule;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
+import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.NullScanOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.optimizer.physical.SparkMapJoinResolver;
@@ -278,6 +279,12 @@ public class SparkCompiler extends TaskC
LOG.debug("Skipping null scan query optimization");
}
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
+ physicalCtx = new MetadataOnlyOptimizer().resolve(physicalCtx);
+ } else {
+ LOG.debug("Skipping metadata only query optimization");
+ }
+
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CHECK_CROSS_PRODUCT)) {
physicalCtx = new CrossProductCheck().resolve(physicalCtx);
} else {
Added: hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out?rev=1641640&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out (added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_partition_metadataonly.q.out Tue Nov 25 15:44:24 2014
@@ -0,0 +1,608 @@
+PREHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select ds from srcpart where hr=11 and ds='2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '2008-04-08' (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ ListSink
+
+PREHOOK: query: select ds from srcpart where hr=11 and ds='2008-04-08'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select ds from srcpart where hr=11 and ds='2008-04-08'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+2008-04-08
+PREHOOK: query: explain select distinct hr from srcpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select distinct hr from srcpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: hr (type: string)
+ outputColumnNames: hr
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: hr (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select distinct hr from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select distinct hr from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+11
+12