You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/10/29 07:40:54 UTC
svn commit: r1635062 - in /pig/branches/branch-0.14: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/
Author: rohini
Date: Wed Oct 29 06:40:54 2014
New Revision: 1635062
URL: http://svn.apache.org/r1635062
Log:
PIG-4245: 1-1 edge vertices should use same jvm opts (rohini)
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Wed Oct 29 06:40:54 2014
@@ -92,6 +92,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4245: 1-1 edge vertices should use same jvm opts (rohini)
+
PIG-4252: Tez container reuse fail when using script udf (daijy)
PIG-4241: Auto local mode mistakenly converts large jobs to local mode when using with Hive tables (cheolsoo)
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Wed Oct 29 06:40:54 2014
@@ -204,8 +204,7 @@ public class TezDagBuilder extends TezOp
Vertex to = null;
try {
if (!tezOp.isVertexGroup()) {
- boolean isMap = (predecessors == null || predecessors.isEmpty()) ? true : false;
- to = newVertex(tezOp, isMap);
+ to = newVertex(tezOp);
dag.addVertex(to);
} else {
// For union, we construct VertexGroup after iterating the
@@ -409,7 +408,7 @@ public class TezDagBuilder extends TezOp
.serialize(new byte[] { combRearrange.getKeyType() }));
}
- private Vertex newVertex(TezOperator tezOp, boolean isMap) throws IOException,
+ private Vertex newVertex(TezOperator tezOp) throws IOException,
ClassNotFoundException, InterruptedException {
ProcessorDescriptor procDesc = ProcessorDescriptor.create(
tezOp.getProcessorName());
@@ -597,10 +596,10 @@ public class TezDagBuilder extends TezOp
procDesc.setUserPayload(userPayload);
Vertex vertex = Vertex.create(tezOp.getOperatorKey().toString(), procDesc, tezOp.getVertexParallelism(),
- isMap ? MRHelpers.getResourceForMRMapper(globalConf) : MRHelpers.getResourceForMRReducer(globalConf));
+ tezOp.isUseMRMapSettings() ? MRHelpers.getResourceForMRMapper(globalConf) : MRHelpers.getResourceForMRReducer(globalConf));
Map<String, String> taskEnv = new HashMap<String, String>();
- MRHelpers.updateEnvBasedOnMRTaskEnv(globalConf, taskEnv, isMap);
+ MRHelpers.updateEnvBasedOnMRTaskEnv(globalConf, taskEnv, tezOp.isUseMRMapSettings());
vertex.setTaskEnvironment(taskEnv);
// All these classes are @InterfaceAudience.Private in Hadoop. Switch to Tez methods in TEZ-1012
@@ -613,7 +612,7 @@ public class TezDagBuilder extends TezOp
MRApps.setupDistributedCache(globalConf, localResources);
vertex.addTaskLocalFiles(localResources);
- vertex.setTaskLaunchCmdOpts(isMap ? MRHelpers.getJavaOptsForMRMapper(globalConf)
+ vertex.setTaskLaunchCmdOpts(tezOp.isUseMRMapSettings() ? MRHelpers.getJavaOptsForMRMapper(globalConf)
: MRHelpers.getJavaOptsForMRReducer(globalConf));
log.info("For vertex - " + tezOp.getOperatorKey().toString()
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Wed Oct 29 06:40:54 2014
@@ -264,6 +264,9 @@ public class TezLauncher extends Launche
public boolean notifyFinishedOrFailed() {
DAGStatus dagStatus = runningJob.getDAGStatus();
+ if (dagStatus == null) {
+ return false;
+ }
if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
Map<Enum, Long> warningAggMap = new HashMap<Enum, Long>();
for (Vertex v : runningJob.getDAG().getVertices()) {
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java Wed Oct 29 06:40:54 2014
@@ -416,6 +416,7 @@ public class TezCompiler extends PhyPlan
// should be added to the tezPlan.
curTezOp = getTezOp();
curTezOp.plan.add(op);
+ curTezOp.setUseMRMapSettings(true);
if (op !=null && op instanceof POLoad) {
if (((POLoad)op).getLFile()!=null && ((POLoad)op).getLFile().getFuncSpec()!=null)
curTezOp.UDFs.add(((POLoad)op).getLFile().getFuncSpec().toString());
@@ -880,6 +881,7 @@ public class TezCompiler extends PhyPlan
public void visitLoad(POLoad op) throws VisitorException {
try {
nonBlocking(op);
+ curTezOp.setUseMRMapSettings(true);
phyToTezOpMap.put(op, curTezOp);
} catch (Exception e) {
int errCode = 2034;
@@ -1417,6 +1419,7 @@ public class TezCompiler extends PhyPlan
// Connect counterOper vertex to rankOper vertex by 1-1 edge
rankOper.setRequestedParallelismByReference(counterOper);
TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, counterOper, rankOper);
+ rankOper.setUseMRMapSettings(counterOper.isUseMRMapSettings());
TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
counterTez.setTuplesOutputKey(rankOper.getOperatorKey().toString());
rankTez.setTuplesInputKey(counterOper.getOperatorKey().toString());
@@ -1652,6 +1655,7 @@ public class TezCompiler extends PhyPlan
pr.setOutputKey(joinJobs[2].getOperatorKey().toString());
TezEdgeDescriptor edge = joinJobs[0].inEdges.get(prevOp.getOperatorKey());
+ joinJobs[0].setUseMRMapSettings(prevOp.isUseMRMapSettings());
// TODO: Convert to unsorted shuffle after TEZ-661
// Use 1-1 edge
edge.dataMovementType = DataMovementType.ONE_TO_ONE;
@@ -2234,6 +2238,7 @@ public class TezCompiler extends PhyPlan
TezOperator[] sortOpers = getSortJobs(prevOper, lr, op, keyType, fields);
TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, prevOper, sortOpers[0]);
+ sortOpers[0].setUseMRMapSettings(prevOper.isUseMRMapSettings());
// Use 1-1 edge
edge.dataMovementType = DataMovementType.ONE_TO_ONE;
@@ -2418,6 +2423,9 @@ public class TezCompiler extends PhyPlan
unionInput.addInputKey(prevTezOp.getOperatorKey().toString());
prevTezOp.plan.addAsLeaf(outputs[i]);
prevTezOp.setClosed(true);
+ if (prevTezOp.isUseMRMapSettings()) {
+ unionTezOp.setUseMRMapSettings(true);
+ }
}
curTezOp = unionTezOp;
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java Wed Oct 29 06:40:54 2014
@@ -125,8 +125,10 @@ public class TezOperator extends Operato
private String crossKey = null;
+ private boolean useMRMapSettings = false;
+
// Types of blocking operators. For now, we only support the following ones.
- private static enum OPER_FEATURE {
+ public static enum OPER_FEATURE {
// Indicate if this job is a merge indexer
INDEXER,
// Indicate if this job is a sampling job
@@ -396,6 +398,17 @@ public class TezOperator extends Operato
feature.set(OPER_FEATURE.NATIVE.ordinal());
}
+ public void copyFeatures(TezOperator copyFrom, List<OPER_FEATURE> excludeFeatures) {
+ for (OPER_FEATURE opf : OPER_FEATURE.values()) {
+ if (excludeFeatures != null && excludeFeatures.contains(opf)) {
+ continue;
+ }
+ if (copyFrom.feature.get(opf.ordinal())) {
+ feature.set(opf.ordinal());
+ }
+ }
+ }
+
public void setNeedEstimatedQuantile(boolean needEstimateParallelism) {
this.needEstimateParallelism = needEstimateParallelism;
}
@@ -550,6 +563,14 @@ public class TezOperator extends Operato
return crossKey;
}
+ public boolean isUseMRMapSettings() {
+ return useMRMapSettings;
+ }
+
+ public void setUseMRMapSettings(boolean useMRMapSettings) {
+ this.useMRMapSettings = useMRMapSettings;
+ }
+
public int getVertexParallelism() {
return vertexParallelism;
}
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java Wed Oct 29 06:40:54 2014
@@ -189,8 +189,6 @@ public class MultiQueryOptimizerTez exte
parentOper.outEdges.put(entry.getKey(), entry.getValue());
}
}
- if (subPlanOper.isSampler()) {
- parentOper.markSampler();
- }
+ parentOper.copyFeatures(subPlanOper, null);
}
}
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java Wed Oct 29 06:40:54 2014
@@ -18,6 +18,7 @@
package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
@@ -30,6 +31,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator.OPER_FEATURE;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator.VertexGroupInfo;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
@@ -265,9 +267,7 @@ public class UnionOptimizer extends TezO
pred.setUseSecondaryKey(unionOp.isUseSecondaryKey());
pred.UDFs.addAll(unionOp.UDFs);
pred.scalars.addAll(unionOp.scalars);
- if (unionOp.isSampler()) {
- pred.markSampler();
- }
+ pred.copyFeatures(unionOp, Arrays.asList(new OPER_FEATURE[]{OPER_FEATURE.UNION}));
}
public static PhysicalPlan getUnionPredPlanFromSplit(PhysicalPlan plan, String unionOpKey) throws VisitorException {