You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/11/05 16:05:25 UTC
svn commit: r1636888 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/mr/ optimizer/ optimizer/physical/ plan/
Author: xuefu
Date: Wed Nov 5 15:05:24 2014
New Revision: 1636888
URL: http://svn.apache.org/r1636888
Log:
HIVE-8718: Refactoring: move mapLocalWork field from MapWork to BaseWork (Reviewed by Szehon)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Wed Nov 5 15:05:24 2014
@@ -329,7 +329,7 @@ public class ExecDriver extends Task<Map
}
try{
- MapredLocalWork localwork = mWork.getMapLocalWork();
+ MapredLocalWork localwork = mWork.getMapRedLocalWork();
if (localwork != null && localwork.hasStagedAlias()) {
if (!ShimLoader.getHadoopShims().isLocalMode(job)) {
Path localPath = localwork.getTmpPath();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java Wed Nov 5 15:05:24 2014
@@ -122,7 +122,7 @@ public class ExecMapper extends MapReduc
mo.setChildren(job);
l4j.info(mo.dump(0));
// initialize map local work
- localWork = mrwork.getMapLocalWork();
+ localWork = mrwork.getMapRedLocalWork();
execContext.setLocalWork(localWork);
MapredContext.init(true, new JobConf(jc));
@@ -214,7 +214,7 @@ public class ExecMapper extends MapReduc
}
if (fetchOperators != null) {
- MapredLocalWork localWork = mo.getConf().getMapLocalWork();
+ MapredLocalWork localWork = mo.getConf().getMapRedLocalWork();
for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
Operator<? extends OperatorDesc> forwardOp = localWork
.getAliasToWork().get(entry.getKey());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Wed Nov 5 15:05:24 2014
@@ -716,7 +716,7 @@ public final class GenMapRedUtils {
plan.getAliasToWork().put(alias_id, topOp);
} else {
// populate local work if needed
- MapredLocalWork localPlan = plan.getMapLocalWork();
+ MapredLocalWork localPlan = plan.getMapRedLocalWork();
if (localPlan == null) {
localPlan = new MapredLocalWork(
new LinkedHashMap<String, Operator<? extends OperatorDesc>>(),
@@ -735,7 +735,7 @@ public final class GenMapRedUtils {
localPlan.getAliasToFetchWork().put(alias_id,
new FetchWork(tblDir, tblDesc));
}
- plan.setMapLocalWork(localPlan);
+ plan.setMapRedLocalWork(localPlan);
}
}
@@ -770,7 +770,7 @@ public final class GenMapRedUtils {
plan.getAliasToWork().put(alias, topOp);
} else {
// populate local work if needed
- MapredLocalWork localPlan = plan.getMapLocalWork();
+ MapredLocalWork localPlan = plan.getMapRedLocalWork();
if (localPlan == null) {
localPlan = new MapredLocalWork(
new LinkedHashMap<String, Operator<? extends OperatorDesc>>(),
@@ -781,7 +781,7 @@ public final class GenMapRedUtils {
assert localPlan.getAliasToFetchWork().get(alias) == null;
localPlan.getAliasToWork().put(alias, topOp);
localPlan.getAliasToFetchWork().put(alias, new FetchWork(new Path(alias), tt_desc));
- plan.setMapLocalWork(localPlan);
+ plan.setMapRedLocalWork(localPlan);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinFactory.java Wed Nov 5 15:05:24 2014
@@ -84,7 +84,7 @@ public final class MapJoinFactory {
Map<String, Map<String, List<String>>> aliasBucketFileNameMapping =
currMapJoinOp.getConf().getAliasBucketFileNameMapping();
if (aliasBucketFileNameMapping != null) {
- MapredLocalWork localPlan = plan.getMapLocalWork();
+ MapredLocalWork localPlan = plan.getMapRedLocalWork();
if (localPlan == null) {
if (currMapJoinOp instanceof SMBMapJoinOperator) {
localPlan = ((SMBMapJoinOperator) currMapJoinOp).getConf().getLocalWork();
@@ -106,10 +106,10 @@ public final class MapJoinFactory {
}
if (currMapJoinOp instanceof SMBMapJoinOperator) {
- plan.setMapLocalWork(null);
+ plan.setMapRedLocalWork(null);
((SMBMapJoinOperator) currMapJoinOp).getConf().setLocalWork(localPlan);
} else {
- plan.setMapLocalWork(localPlan);
+ plan.setMapRedLocalWork(localPlan);
}
BucketMapJoinContext bucketMJCxt = new BucketMapJoinContext();
localPlan.setBucketMapjoinContext(bucketMJCxt);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Wed Nov 5 15:05:24 2014
@@ -212,7 +212,7 @@ public class MapJoinProcessor implements
}
// set up local work
- newWork.getMapWork().setMapLocalWork(newLocalWork);
+ newWork.getMapWork().setMapRedLocalWork(newLocalWork);
// remove reducer
newWork.setReduceWork(null);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Wed Nov 5 15:05:24 2014
@@ -257,8 +257,8 @@ public class CommonJoinTaskDispatcher ex
}
}
- MapredLocalWork mapJoinLocalWork = mapJoinMapWork.getMapLocalWork();
- MapredLocalWork childLocalWork = childMapWork.getMapLocalWork();
+ MapredLocalWork mapJoinLocalWork = mapJoinMapWork.getMapRedLocalWork();
+ MapredLocalWork childLocalWork = childMapWork.getMapRedLocalWork();
if ((mapJoinLocalWork != null && mapJoinLocalWork.getBucketMapjoinContext() != null) ||
(childLocalWork != null && childLocalWork.getBucketMapjoinContext() != null)) {
@@ -328,7 +328,7 @@ public class CommonJoinTaskDispatcher ex
// Step 2.3: Fill up stuff in local work
if (mapJoinLocalWork != null) {
if (childLocalWork == null) {
- childMapWork.setMapLocalWork(mapJoinLocalWork);
+ childMapWork.setMapRedLocalWork(mapJoinLocalWork);
} else {
childLocalWork.getAliasToFetchWork().putAll(mapJoinLocalWork.getAliasToFetchWork());
childLocalWork.getAliasToWork().putAll(mapJoinLocalWork.getAliasToWork());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java Wed Nov 5 15:05:24 2014
@@ -310,7 +310,7 @@ public final class GenMRSkewJoinProcesso
new FetchWork(tblDir, tableDescList.get(small_alias)));
}
- newPlan.setMapLocalWork(localPlan);
+ newPlan.setMapRedLocalWork(localPlan);
// construct a map join and set it as the child operator of tblScan_op
MapJoinOperator mapJoinOp = (MapJoinOperator) OperatorFactory
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java Wed Nov 5 15:05:24 2014
@@ -100,7 +100,7 @@ public class MapJoinResolver implements
ConditionalTask conditionalTask) throws SemanticException {
// get current mapred work and its local work
MapredWork mapredWork = (MapredWork) currTask.getWork();
- MapredLocalWork localwork = mapredWork.getMapWork().getMapLocalWork();
+ MapredLocalWork localwork = mapredWork.getMapWork().getMapRedLocalWork();
if (localwork != null) {
// get the context info and set up the shared tmp URI
Context ctx = physicalContext.getContext();
@@ -135,7 +135,7 @@ public class MapJoinResolver implements
MapredLocalWork newLocalWork = localwork.extractDirectWorks(
localMapJoinProcCtx.getDirectWorks());
newLocalWork.setDummyParentOp(dummyOps);
- mapredWork.getMapWork().setMapLocalWork(newLocalWork);
+ mapredWork.getMapWork().setMapRedLocalWork(newLocalWork);
if (localwork.getAliasToFetchWork().isEmpty()) {
// no alias to stage.. no local task
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Wed Nov 5 15:05:24 2014
@@ -45,6 +45,8 @@ public abstract class BaseWork extends A
int tag;
private final List<String> sortColNames = new ArrayList<String>();
+ private MapredLocalWork mrLocalWork;
+
public BaseWork() {}
public BaseWork(String name) {
@@ -132,6 +134,22 @@ public abstract class BaseWork extends A
this.allColumnVectorMaps = allColumnVectorMaps;
}
+ /**
+ * @return the mapredLocalWork
+ */
+ @Explain(displayName = "Local Work")
+ public MapredLocalWork getMapRedLocalWork() {
+ return mrLocalWork;
+ }
+
+ /**
+ * @param mapLocalWork
+ * the mapredLocalWork to set
+ */
+ public void setMapRedLocalWork(final MapredLocalWork mapLocalWork) {
+ this.mrLocalWork = mapLocalWork;
+ }
+
@Override
public void setVectorMode(boolean vectorMode) {
this.vectorMode = vectorMode;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1636888&r1=1636887&r2=1636888&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Wed Nov 5 15:05:24 2014
@@ -85,7 +85,6 @@ public class MapWork extends BaseWork {
private final Map<String, List<SortCol>> sortedColsByDirectory =
new HashMap<String, List<SortCol>>();
- private MapredLocalWork mapLocalWork;
private Path tmpHDFSPath;
private String inputformat;
@@ -194,6 +193,8 @@ public class MapWork extends BaseWork {
entry.getValue().deriveBaseFileName(entry.getKey());
}
}
+
+ MapredLocalWork mapLocalWork = getMapRedLocalWork();
if (mapLocalWork != null) {
mapLocalWork.deriveExplainAttributes();
}
@@ -240,23 +241,6 @@ public class MapWork extends BaseWork {
this.aliasToWork = aliasToWork;
}
- /**
- * @return the mapredLocalWork
- */
- @Explain(displayName = "Local Work")
- public MapredLocalWork getMapLocalWork() {
- return mapLocalWork;
- }
-
- /**
- * @param mapLocalWork
- * the mapredLocalWork to set
- */
- public void setMapLocalWork(final MapredLocalWork mapLocalWork) {
- this.mapLocalWork = mapLocalWork;
- }
-
-
@Explain(displayName = "Split Sample", normalExplain = false)
public HashMap<String, SplitSample> getNameToSplitSample() {
return nameToSplitSample;