You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/10/29 07:40:54 UTC

svn commit: r1635062 - in /pig/branches/branch-0.14: ./ src/org/apache/pig/backend/hadoop/executionengine/tez/ src/org/apache/pig/backend/hadoop/executionengine/tez/plan/ src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/

Author: rohini
Date: Wed Oct 29 06:40:54 2014
New Revision: 1635062

URL: http://svn.apache.org/r1635062
Log:
PIG-4245: 1-1 edge vertices should use same jvm opts (rohini)

Modified:
    pig/branches/branch-0.14/CHANGES.txt
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
    pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java

Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Wed Oct 29 06:40:54 2014
@@ -92,6 +92,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-4245: 1-1 edge vertices should use same jvm opts (rohini)
+
 PIG-4252: Tez container reuse fail when using script udf (daijy)
 
 PIG-4241: Auto local mode mistakenly converts large jobs to local mode when using with Hive tables (cheolsoo)

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Wed Oct 29 06:40:54 2014
@@ -204,8 +204,7 @@ public class TezDagBuilder extends TezOp
         Vertex to = null;
         try {
             if (!tezOp.isVertexGroup()) {
-                boolean isMap = (predecessors == null || predecessors.isEmpty()) ? true : false;
-                to = newVertex(tezOp, isMap);
+                to = newVertex(tezOp);
                 dag.addVertex(to);
             } else {
                 // For union, we construct VertexGroup after iterating the
@@ -409,7 +408,7 @@ public class TezDagBuilder extends TezOp
                 .serialize(new byte[] { combRearrange.getKeyType() }));
     }
 
-    private Vertex newVertex(TezOperator tezOp, boolean isMap) throws IOException,
+    private Vertex newVertex(TezOperator tezOp) throws IOException,
             ClassNotFoundException, InterruptedException {
         ProcessorDescriptor procDesc = ProcessorDescriptor.create(
                 tezOp.getProcessorName());
@@ -597,10 +596,10 @@ public class TezDagBuilder extends TezOp
         procDesc.setUserPayload(userPayload);
 
         Vertex vertex = Vertex.create(tezOp.getOperatorKey().toString(), procDesc, tezOp.getVertexParallelism(),
-                isMap ? MRHelpers.getResourceForMRMapper(globalConf) : MRHelpers.getResourceForMRReducer(globalConf));
+                tezOp.isUseMRMapSettings() ? MRHelpers.getResourceForMRMapper(globalConf) : MRHelpers.getResourceForMRReducer(globalConf));
 
         Map<String, String> taskEnv = new HashMap<String, String>();
-        MRHelpers.updateEnvBasedOnMRTaskEnv(globalConf, taskEnv, isMap);
+        MRHelpers.updateEnvBasedOnMRTaskEnv(globalConf, taskEnv, tezOp.isUseMRMapSettings());
         vertex.setTaskEnvironment(taskEnv);
 
         // All these classes are @InterfaceAudience.Private in Hadoop. Switch to Tez methods in TEZ-1012
@@ -613,7 +612,7 @@ public class TezDagBuilder extends TezOp
         MRApps.setupDistributedCache(globalConf, localResources);
         vertex.addTaskLocalFiles(localResources);
 
-        vertex.setTaskLaunchCmdOpts(isMap ? MRHelpers.getJavaOptsForMRMapper(globalConf)
+        vertex.setTaskLaunchCmdOpts(tezOp.isUseMRMapSettings() ? MRHelpers.getJavaOptsForMRMapper(globalConf)
                 : MRHelpers.getJavaOptsForMRReducer(globalConf));
 
         log.info("For vertex - " + tezOp.getOperatorKey().toString()

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Wed Oct 29 06:40:54 2014
@@ -264,6 +264,9 @@ public class TezLauncher extends Launche
 
         public boolean notifyFinishedOrFailed() {
             DAGStatus dagStatus = runningJob.getDAGStatus();
+            if (dagStatus == null) {
+                return false;
+            }
             if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
                 Map<Enum, Long> warningAggMap = new HashMap<Enum, Long>();
                 for (Vertex v : runningJob.getDAG().getVertices()) {

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java Wed Oct 29 06:40:54 2014
@@ -416,6 +416,7 @@ public class TezCompiler extends PhyPlan
             // should be added to the tezPlan.
             curTezOp = getTezOp();
             curTezOp.plan.add(op);
+            curTezOp.setUseMRMapSettings(true);
             if (op !=null && op instanceof POLoad) {
                 if (((POLoad)op).getLFile()!=null && ((POLoad)op).getLFile().getFuncSpec()!=null)
                     curTezOp.UDFs.add(((POLoad)op).getLFile().getFuncSpec().toString());
@@ -880,6 +881,7 @@ public class TezCompiler extends PhyPlan
     public void visitLoad(POLoad op) throws VisitorException {
         try {
             nonBlocking(op);
+            curTezOp.setUseMRMapSettings(true);
             phyToTezOpMap.put(op, curTezOp);
         } catch (Exception e) {
             int errCode = 2034;
@@ -1417,6 +1419,7 @@ public class TezCompiler extends PhyPlan
             // Connect counterOper vertex to rankOper vertex by 1-1 edge
             rankOper.setRequestedParallelismByReference(counterOper);
             TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, counterOper, rankOper);
+            rankOper.setUseMRMapSettings(counterOper.isUseMRMapSettings());
             TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
             counterTez.setTuplesOutputKey(rankOper.getOperatorKey().toString());
             rankTez.setTuplesInputKey(counterOper.getOperatorKey().toString());
@@ -1652,6 +1655,7 @@ public class TezCompiler extends PhyPlan
             pr.setOutputKey(joinJobs[2].getOperatorKey().toString());
 
             TezEdgeDescriptor edge = joinJobs[0].inEdges.get(prevOp.getOperatorKey());
+            joinJobs[0].setUseMRMapSettings(prevOp.isUseMRMapSettings());
             // TODO: Convert to unsorted shuffle after TEZ-661
             // Use 1-1 edge
             edge.dataMovementType = DataMovementType.ONE_TO_ONE;
@@ -2234,6 +2238,7 @@ public class TezCompiler extends PhyPlan
             TezOperator[] sortOpers = getSortJobs(prevOper, lr, op, keyType, fields);
 
             TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, prevOper, sortOpers[0]);
+            sortOpers[0].setUseMRMapSettings(prevOper.isUseMRMapSettings());
 
             // Use 1-1 edge
             edge.dataMovementType = DataMovementType.ONE_TO_ONE;
@@ -2418,6 +2423,9 @@ public class TezCompiler extends PhyPlan
                 unionInput.addInputKey(prevTezOp.getOperatorKey().toString());
                 prevTezOp.plan.addAsLeaf(outputs[i]);
                 prevTezOp.setClosed(true);
+                if (prevTezOp.isUseMRMapSettings()) {
+                    unionTezOp.setUseMRMapSettings(true);
+                }
             }
 
             curTezOp = unionTezOp;

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperator.java Wed Oct 29 06:40:54 2014
@@ -125,8 +125,10 @@ public class TezOperator extends Operato
 
     private String crossKey = null;
 
+    private boolean useMRMapSettings = false;
+
     // Types of blocking operators. For now, we only support the following ones.
-    private static enum OPER_FEATURE {
+    public static enum OPER_FEATURE {
         // Indicate if this job is a merge indexer
         INDEXER,
         // Indicate if this job is a sampling job
@@ -396,6 +398,17 @@ public class TezOperator extends Operato
         feature.set(OPER_FEATURE.NATIVE.ordinal());
     }
 
+    public void copyFeatures(TezOperator copyFrom, List<OPER_FEATURE> excludeFeatures) {
+        for (OPER_FEATURE opf : OPER_FEATURE.values()) {
+            if (excludeFeatures != null && excludeFeatures.contains(opf)) {
+                continue;
+            }
+            if (copyFrom.feature.get(opf.ordinal())) {
+                feature.set(opf.ordinal());
+            }
+        }
+    }
+
     public void setNeedEstimatedQuantile(boolean needEstimateParallelism) {
         this.needEstimateParallelism = needEstimateParallelism;
     }
@@ -550,6 +563,14 @@ public class TezOperator extends Operato
         return crossKey;
     }
 
+    public boolean isUseMRMapSettings() {
+        return useMRMapSettings;
+    }
+
+    public void setUseMRMapSettings(boolean useMRMapSettings) {
+        this.useMRMapSettings = useMRMapSettings;
+    }
+
     public int getVertexParallelism() {
         return vertexParallelism;
     }

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/MultiQueryOptimizerTez.java Wed Oct 29 06:40:54 2014
@@ -189,8 +189,6 @@ public class MultiQueryOptimizerTez exte
                 parentOper.outEdges.put(entry.getKey(), entry.getValue());
             }
         }
-        if (subPlanOper.isSampler()) {
-            parentOper.markSampler();
-        }
+        parentOper.copyFeatures(subPlanOper, null);
     }
 }

Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java?rev=1635062&r1=1635061&r2=1635062&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/UnionOptimizer.java Wed Oct 29 06:40:54 2014
@@ -18,6 +18,7 @@
 package org.apache.pig.backend.hadoop.executionengine.tez.plan.optimizer;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map.Entry;
@@ -30,6 +31,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOpPlanVisitor;
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperPlan;
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator.OPER_FEATURE;
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezOperator.VertexGroupInfo;
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
 import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POValueOutputTez;
@@ -265,9 +267,7 @@ public class UnionOptimizer extends TezO
         pred.setUseSecondaryKey(unionOp.isUseSecondaryKey());
         pred.UDFs.addAll(unionOp.UDFs);
         pred.scalars.addAll(unionOp.scalars);
-        if (unionOp.isSampler()) {
-            pred.markSampler();
-        }
+        pred.copyFeatures(unionOp, Arrays.asList(new OPER_FEATURE[]{OPER_FEATURE.UNION}));
     }
 
     public static PhysicalPlan getUnionPredPlanFromSplit(PhysicalPlan plan, String unionOpKey) throws VisitorException {