You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/12/03 19:07:51 UTC

svn commit: r1547505 [2/2] - in /pig/branches/tez: src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/backend/hadoop/executionengine/...

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Tue Dec  3 18:07:50 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.Input
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -51,6 +52,7 @@ import org.apache.pig.backend.hadoop.HDa
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PhyPlanSetter;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBigDecimalRawComparator;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBigIntegerRawComparator;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBooleanRawComparator;
@@ -87,15 +89,16 @@ import org.apache.pig.impl.util.ObjectSe
 import org.apache.pig.impl.util.UDFContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.mapreduce.combine.MRCombiner;
-import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
+import org.apache.tez.mapreduce.common.MRInputSplitDistributor;
+import org.apache.tez.mapreduce.hadoop.InputSplitInfo;
 import org.apache.tez.mapreduce.hadoop.MRHelpers;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 import org.apache.tez.mapreduce.input.MRInput;
@@ -108,7 +111,7 @@ import org.apache.tez.mapreduce.partitio
 public class TezDagBuilder extends TezOpPlanVisitor {
     private static final Log log = LogFactory.getLog(TezJobControlCompiler.class);
 
-    private DAG dag;
+    private TezDAG dag;
     private Map<String, LocalResource> localResources;
     private PigContext pc;
     private Configuration globalConf;
@@ -116,7 +119,7 @@ public class TezDagBuilder extends TezOp
     private String scope;
     private NodeIdGenerator nig;
 
-    public TezDagBuilder(PigContext pc, TezOperPlan plan, DAG dag,
+    public TezDagBuilder(PigContext pc, TezOperPlan plan, TezDAG dag,
             Map<String, LocalResource> localResources) {
         super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan));
         this.pc = pc;
@@ -134,7 +137,7 @@ public class TezDagBuilder extends TezOp
         try {
             to = newVertex(tezOp);
             dag.addVertex(to);
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new VisitorException("Cannot create vertex for "
                     + tezOp.name(), e);
         }
@@ -146,7 +149,7 @@ public class TezDagBuilder extends TezOp
             for (TezOperator predecessor : predecessors) {
                 // Since this is a dependency order walker, predecessor vertices
                 // must have already been created.
-                Vertex from = dag.getVertex(predecessor.name());
+                Vertex from = dag.getVertex(predecessor.getOperatorKey().toString());
                 EdgeProperty prop = null;
                 try {
                     prop = newEdge(predecessor, tezOp);
@@ -176,19 +179,34 @@ public class TezDagBuilder extends TezOp
         InputDescriptor in = new InputDescriptor(edge.inputClassName);
         OutputDescriptor out = new OutputDescriptor(edge.outputClassName);
 
+        Configuration conf = new Configuration(false);
         if (!combinePlan.isEmpty()) {
-            Configuration conf = new Configuration();
-            addCombiner(combinePlan, conf);
-            in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
-            out.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+            addCombiner(combinePlan, to, conf);
         }
 
+        List<POLocalRearrangeTez> lrs = PlanHelper.getPhysicalOperators(from.plan,
+                POLocalRearrangeTez.class);
+
+        for (POLocalRearrangeTez lr : lrs) {
+            if (lr.getOutputKey().equals(to.getOperatorKey().toString())) {
+                byte keyType = lr.getKeyType();
+                setIntermediateInputKeyValue(keyType, conf);
+                setIntermediateOutputKeyValue(keyType, conf);
+                conf.set("pig.reduce.key.type", Byte.toString(keyType));
+                break;
+            }
+        }
+
+        in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+        out.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+
         return new EdgeProperty(edge.dataMovementType, edge.dataSourceType,
                 edge.schedulingType, out, in);
     }
 
-    private void addCombiner(PhysicalPlan combinePlan, Configuration conf) throws IOException {
-        POPackage combPack = (POPackage)combinePlan.getRoots().get(0);
+    private void addCombiner(PhysicalPlan combinePlan, TezOperator pkgTezOp,
+            Configuration conf) throws IOException {
+        POPackage combPack = (POPackage) combinePlan.getRoots().get(0);
         setIntermediateInputKeyValue(combPack.getPkgr().getKeyType(), conf);
 
         POLocalRearrange combRearrange = (POLocalRearrange) combinePlan
@@ -196,7 +214,7 @@ public class TezDagBuilder extends TezOp
         setIntermediateOutputKeyValue(combRearrange.getKeyType(), conf);
 
         LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(
-                combinePlan, combPack);
+                combinePlan, pkgTezOp, combPack);
         lrDiscoverer.visit();
 
         combinePlan.remove(combPack);
@@ -212,21 +230,21 @@ public class TezDagBuilder extends TezOp
                 .serialize(new byte[] { combRearrange.getKeyType() }));
     }
 
-    private Vertex newVertex(TezOperator tezOp) throws IOException {
+    private Vertex newVertex(TezOperator tezOp) throws IOException,
+            ClassNotFoundException, InterruptedException {
         ProcessorDescriptor procDesc = new ProcessorDescriptor(
                 tezOp.getProcessorName());
 
-        // Pass physical plans to vertex as user payload.
-        Configuration payloadConf = new Configuration();
-        // We won't actually use this job, but we need it to talk with the Load
-        // Store funcs
+        // We won't actually use this job, but we need it to talk with the Load Store funcs
         @SuppressWarnings("deprecation")
-        Job job = new Job(payloadConf);
+        Job job = new Job();
 
-        ArrayList<POStore> storeLocations = new ArrayList<POStore>();
-        Path tmpLocation = null;
+        // Pass physical plans to vertex as user payload.
+        Configuration payloadConf = job.getConfiguration();
 
         List<POLoad> loads = processLoads(tezOp, payloadConf, job);
+        LinkedList<POStore> stores = processStores(tezOp, payloadConf, job);
+
         payloadConf.set("pig.pigContext", ObjectSerializer.serialize(pc));
 
         payloadConf.set("udf.import.list",
@@ -235,87 +253,6 @@ public class TezDagBuilder extends TezOp
         payloadConf.setClass("mapreduce.inputformat.class",
                 PigInputFormat.class, InputFormat.class);
 
-        // We need to remove all the POStores from the exec plan and later add
-        // them as outputs of the vertex
-        LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
-                tezOp.plan, POStore.class);
-
-        for (POStore st : stores) {
-            storeLocations.add(st);
-            StoreFuncInterface sFunc = st.getStoreFunc();
-            sFunc.setStoreLocation(st.getSFile().getFileName(), job);
-        }
-
-        if (stores.size() == 1) {
-            POStore st = stores.get(0);
-            if (!pc.inIllustrator)
-                tezOp.plan.remove(st);
-
-            // set out filespecs
-            String outputPathString = st.getSFile().getFileName();
-            if (!outputPathString.contains("://")
-                    || outputPathString.startsWith("hdfs://")) {
-                payloadConf.set("pig.streaming.log.dir", new Path(
-                        outputPathString, JobControlCompiler.LOG_DIR)
-                        .toString());
-            } else {
-                String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
-                        .toString();
-                tmpLocation = new Path(tmpLocationStr);
-                payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
-                        JobControlCompiler.LOG_DIR).toString());
-            }
-            payloadConf.set("pig.streaming.task.output.dir", outputPathString);
-        } else if (stores.size() > 0) { // multi store case
-            log.info("Setting up multi store job");
-            String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
-                    .toString();
-            tmpLocation = new Path(tmpLocationStr);
-
-            boolean disableCounter = payloadConf.getBoolean(
-                    "pig.disable.counter", false);
-            if (disableCounter) {
-                log.info("Disable Pig custom output counters");
-            }
-            int idx = 0;
-            for (POStore sto : storeLocations) {
-                sto.setDisableCounter(disableCounter);
-                sto.setMultiStore(true);
-                sto.setIndex(idx++);
-            }
-
-            payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
-                    JobControlCompiler.LOG_DIR).toString());
-            payloadConf.set("pig.streaming.task.output.dir",
-                    tmpLocation.toString());
-        }
-
-        if (!pc.inIllustrator) {
-            // Unset inputs for POStore, otherwise, exec plan will be
-            // unnecessarily deserialized
-            for (POStore st : stores) {
-                st.setInputs(null);
-                st.setParentPlan(null);
-            }
-            // We put them in the reduce because PigOutputCommitter checks the
-            // ID of the task to see if it's a map, and if not, calls the reduce
-            // committers.
-            payloadConf.set(JobControlCompiler.PIG_MAP_STORES,
-                    ObjectSerializer.serialize(new ArrayList<POStore>()));
-            payloadConf.set(JobControlCompiler.PIG_REDUCE_STORES,
-                    ObjectSerializer.serialize(stores));
-        }
-
-        // For all shuffle outputs, configure the classes
-        List<PhysicalOperator> leaves = tezOp.plan.getLeaves();
-        // TODO: For multiple POLocalRearrange leaves, we need to loop over
-        // leaves and set up per shuffle. i.e. SPLIT + multiple GROUP BY with
-        // different keys.
-        if (leaves.size() == 1 && leaves.get(0) instanceof POLocalRearrange) {
-            byte keyType = ((POLocalRearrange)leaves.get(0)).getKeyType();
-            setIntermediateOutputKeyValue(keyType, payloadConf);
-            payloadConf.set("pig.reduce.key.type", Byte.toString(keyType));
-        }
 
         // Configure the classes for incoming shuffles to this TezOp
         List<PhysicalOperator> roots = tezOp.plan.getRoots();
@@ -382,6 +319,10 @@ public class TezDagBuilder extends TezOp
         payloadConf.setClass("mapreduce.outputformat.class",
                 PigOutputFormat.class, OutputFormat.class);
 
+        // set parent plan in all operators.
+        // currently the parent plan is really used only when POStream, POSplit are present in the plan
+        new PhyPlanSetter(tezOp.plan).visit();
+
         // Serialize the execution plan
         payloadConf.set(PigProcessor.PLAN,
                 ObjectSerializer.serialize(tezOp.plan));
@@ -392,8 +333,17 @@ public class TezDagBuilder extends TezOp
         procDesc.setUserPayload(userPayload);
         // Can only set parallelism here if the parallelism isn't derived from
         // splits
-        int parallelism = (loads.size() == 0) ? tezOp.requestedParallelism : -1;
-        Vertex vertex = new Vertex(tezOp.name(), procDesc, parallelism,
+        int parallelism = tezOp.requestedParallelism;
+        InputSplitInfo inputSplitInfo = null;
+        if (loads != null && loads.size() > 0) {
+            // Not using MRInputAMSplitGenerator because delegation tokens are
+            // fetched in FileInputFormat
+            inputSplitInfo = MRHelpers.generateInputSplitsToMem(payloadConf);
+            // TODO: Can be set to -1 if TEZ-601 gets fixed and getting input
+            // splits can be moved to if(loads) block below
+            parallelism = inputSplitInfo.getNumTasks();
+        }
+        Vertex vertex = new Vertex(tezOp.getOperatorKey().toString(), procDesc, parallelism,
                 Resource.newInstance(tezOp.requestedMemory, tezOp.requestedCpu));
 
         Map<String, String> env = new HashMap<String, String>();
@@ -413,16 +363,55 @@ public class TezDagBuilder extends TezOp
 
             // TODO: These should get the globalConf, or a merged version that
             // keeps settings like pig.maxCombinedSplitSize
+            vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
             vertex.addInput(ld.getOperatorKey().toString(),
                     new InputDescriptor(MRInput.class.getName())
                             .setUserPayload(MRHelpers.createMRInputPayload(
-                                    userPayload, null)),
-                    MRInputAMSplitGenerator.class);
-        }
-        if (!stores.isEmpty()) {
-            vertex.addOutput("PigOutput",
-                    new OutputDescriptor(MROutput.class.getName()));
-        }
+                                    userPayload,
+                                    inputSplitInfo.getSplitsProto())),
+                    MRInputSplitDistributor.class);
+        }
+
+        for (POStore store : stores) {
+
+            ArrayList<POStore> emptyList = new ArrayList<POStore>();
+            ArrayList<POStore> singleStore = new ArrayList<POStore>();
+            singleStore.add(store);
+
+            Configuration outputPayLoad = new Configuration(payloadConf);
+            outputPayLoad.set(JobControlCompiler.PIG_MAP_STORES,
+                    ObjectSerializer.serialize(emptyList));
+            outputPayLoad.set(JobControlCompiler.PIG_REDUCE_STORES,
+                    ObjectSerializer.serialize(singleStore));
+
+            vertex.addOutput(store.getOperatorKey().toString(),
+                    new OutputDescriptor(MROutput.class.getName())
+                            .setUserPayload(TezUtils.createUserPayloadFromConf(outputPayLoad)));
+        }
+
+        if (stores.size() > 0) {
+            new PigOutputFormat().checkOutputSpecs(job);
+        }
+
+        // LoadFunc and StoreFunc add delegation tokens to Job Credentials in
+        // setLocation and setStoreLocation respectively. For eg: HBaseStorage
+        // InputFormat add delegation token in getSplits and OutputFormat in
+        // checkOutputSpecs. For eg: FileInputFormat and FileOutputFormat
+        dag.getCredentials().addAll(job.getCredentials());
+
+        // Add credentials from binary token file and get tokens for namenodes
+        // specified in mapreduce.job.hdfs-servers
+        SecurityHelper.populateTokenCache(job.getConfiguration(), dag.getCredentials());
+
+        // Unlike MR which gets staging dir from RM, tez AM is picking it up
+        // from configuration
+        Path submitJobDir = new Path(payloadConf.get(
+                TezConfiguration.TEZ_AM_STAGING_DIR,
+                TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
+        // Get delegation token for the job submission dir
+        TokenCache.obtainTokensForNamenodes(dag.getCredentials(),
+                new Path[] { submitJobDir }, payloadConf);
+
         return vertex;
     }
 
@@ -495,6 +484,78 @@ public class TezDagBuilder extends TezOp
         return lds;
     }
 
+    private LinkedList<POStore> processStores(TezOperator tezOp,
+            Configuration payloadConf, Job job) throws VisitorException,
+            IOException {
+        LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
+                tezOp.plan, POStore.class);
+        if (stores.size() > 0) {
+
+            ArrayList<POStore> storeLocations = new ArrayList<POStore>();
+            for (POStore st : stores) {
+                storeLocations.add(st);
+                StoreFuncInterface sFunc = st.getStoreFunc();
+                sFunc.setStoreLocation(st.getSFile().getFileName(), job);
+            }
+
+            Path tmpLocation = null;
+            if (stores.size() == 1) {
+                POStore st = stores.get(0);
+
+                // set out filespecs
+                String outputPathString = st.getSFile().getFileName();
+                if (!outputPathString.contains("://")
+                        || outputPathString.startsWith("hdfs://")) {
+                    payloadConf.set("pig.streaming.log.dir", new Path(
+                            outputPathString, JobControlCompiler.LOG_DIR)
+                            .toString());
+                } else {
+                    String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
+                            .toString();
+                    tmpLocation = new Path(tmpLocationStr);
+                    payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
+                            JobControlCompiler.LOG_DIR).toString());
+                }
+                payloadConf.set("pig.streaming.task.output.dir", outputPathString);
+            } else { // multi store case
+                log.info("Setting up multi store job");
+                String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
+                        .toString();
+                tmpLocation = new Path(tmpLocationStr);
+
+                boolean disableCounter = payloadConf.getBoolean(
+                        "pig.disable.counter", false);
+                if (disableCounter) {
+                    log.info("Disable Pig custom output counters");
+                }
+                int idx = 0;
+                for (POStore sto : storeLocations) {
+                    sto.setDisableCounter(disableCounter);
+                    sto.setMultiStore(true);
+                    sto.setIndex(idx++);
+                }
+
+                payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
+                        JobControlCompiler.LOG_DIR).toString());
+                payloadConf.set("pig.streaming.task.output.dir",
+                        tmpLocation.toString());
+            }
+
+            if (!pc.inIllustrator) {
+
+                // We put them in the reduce because PigOutputCommitter checks the
+                // ID of the task to see if it's a map, and if not, calls the reduce
+                // committers.
+                payloadConf.set(JobControlCompiler.PIG_MAP_STORES,
+                        ObjectSerializer.serialize(new ArrayList<POStore>()));
+                payloadConf.set(JobControlCompiler.PIG_REDUCE_STORES,
+                        ObjectSerializer.serialize(stores));
+            }
+
+        }
+        return stores;
+    }
+
     @SuppressWarnings("rawtypes")
     private void setIntermediateInputKeyValue(byte keyType, Configuration conf)
             throws JobCreationException, ExecException {

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java Tue Dec  3 18:07:50 2013
@@ -41,13 +41,13 @@ public class TezJob extends ControlledJo
     private static final Log log = LogFactory.getLog(TezJob.class);
     private DAGStatus dagStatus;
     private Configuration conf;
-    private DAG dag;
+    private TezDAG dag;
     private DAGClient dagClient;
     private Map<String, LocalResource> requestAMResources;
     private TezSession tezSession;
     private boolean reuseSession;
 
-    public TezJob(TezConfiguration conf, DAG dag, Map<String, LocalResource> requestAMResources)
+    public TezJob(TezConfiguration conf, TezDAG dag, Map<String, LocalResource> requestAMResources)
             throws IOException {
         super(conf);
         this.conf = conf;
@@ -67,12 +67,14 @@ public class TezJob extends ControlledJo
     @Override
     public void submit() {
         try {
-            tezSession = TezSessionManager.getSession(conf, requestAMResources);
+            tezSession = TezSessionManager.getSession(conf, requestAMResources, dag.getCredentials());
             log.info("Submitting DAG - Application id: " + tezSession.getApplicationId());
             dagClient = tezSession.submitDAG(dag);
         } catch (Exception e) {
-            if (tezSession!=null) {
-                log.info("Cannot submit DAG - Application id: " + tezSession.getApplicationId(), e);
+            if (tezSession != null) {
+                log.error("Cannot submit DAG - Application id: " + tezSession.getApplicationId(), e);
+            } else {
+                log.error("Cannot submit DAG", e);
             }
             setJobState(ControlledJob.State.FAILED);
             return;

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java Tue Dec  3 18:07:50 2013
@@ -23,13 +23,12 @@ import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.PigException;
 import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
 import org.apache.pig.impl.PigContext;
-import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.TezConfiguration;
 
 /**
@@ -48,10 +47,10 @@ public class TezJobControlCompiler {
         this.tezConf = new TezConfiguration(conf);
     }
 
-    public DAG buildDAG(TezOperPlan tezPlan, Map<String, LocalResource> localResources)
+    public TezDAG buildDAG(TezOperPlan tezPlan, Map<String, LocalResource> localResources)
             throws IOException, YarnException {
         String jobName = pigContext.getProperties().getProperty(PigContext.JOB_NAME, "pig");
-        DAG tezDag = new DAG(jobName);
+        TezDAG tezDag = new TezDAG(jobName);
         TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlan, tezDag, localResources);
         dagBuilder.visit();
         return tezDag;
@@ -78,7 +77,7 @@ public class TezJobControlCompiler {
         TezJobControl jobCtrl = new TezJobControl(grpName, timeToSleep);
 
         try {
-            // A single Tez job always pack only 1 Tez plan. We will track 
+            // A single Tez job always pack only 1 Tez plan. We will track
             // Tez job asynchronously to exploit parallel execution opportunities.
             TezJob job = getJob(tezPlan, planContainer);
             jobCtrl.addJob(job);
@@ -99,7 +98,7 @@ public class TezJobControlCompiler {
             Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
             localResources.putAll(planContainer.getLocalResources());
             localResources.putAll(tezPlan.getLocalExtraResources());
-            DAG tezDag = buildDAG(tezPlan, localResources);
+            TezDAG tezDag = buildDAG(tezPlan, localResources);
             return new TezJob(tezConf, tezDag, planContainer.getLocalResources());
         } catch (Exception e) {
             int errCode = 2017;

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java Tue Dec  3 18:07:50 2013
@@ -62,26 +62,30 @@ public class TezOperator extends Operato
     // Name of the Custom Partitioner used
     String customPartitioner = null;
 
+    // Presence indicates that this TezOper is sub-plan of a POSplit.
+    // Only POStore or POLocalRearrange leaf can be a sub-plan of POSplit
+    private OperatorKey splitOperatorKey = null;
+
     // Indicates that the plan creation is complete
     boolean closed = false;
-    
+
     // Indicate whether we need to split the DAG below the operator
     // The result is two or more DAG connected DAG inside the same plan container
     boolean segmentBelow = false;
-    
+
     // Indicates if this is a limit after a sort
     boolean limitAfterSort = false;
-    
+
     //Indicates if this job is an order by job
     boolean globalSort = false;
-    
+
     //The quantiles file name if globalSort is true
     String quantFile;
-    
+
     //The sort order of the columns;
     //asc is true and desc is false
     boolean[] sortOrder;
-    
+
     // Last POLimit value in this map reduce operator, needed by LimitAdjuster
     // to add additional map reduce operator with 1 reducer after this
     long limit = -1;
@@ -119,14 +123,28 @@ public class TezOperator extends Operato
         v.visitTezOp(this);
     }
 
+    @Override
     public boolean supportsMultipleInputs() {
         return true;
     }
 
+    @Override
     public boolean supportsMultipleOutputs() {
         return true;
     }
 
+    public OperatorKey getSplitOperatorKey() {
+        return splitOperatorKey;
+    }
+
+    public void setSplitOperatorKey(OperatorKey splitOperatorKey) {
+        this.splitOperatorKey = splitOperatorKey;
+    }
+
+    public boolean isSplitSubPlan() {
+        return splitOperatorKey != null;
+    }
+
     public boolean isClosed() {
         return closed;
     }
@@ -216,7 +234,7 @@ public class TezOperator extends Operato
     public boolean needSegmentBelow() {
         return segmentBelow;
     }
-    
+
     public String getQuantFile() {
         return quantFile;
     }
@@ -232,19 +250,19 @@ public class TezOperator extends Operato
             this.sortOrder[i] = sortOrder[i];
         }
     }
-    
+
     public boolean[] getSortOrder() {
         return sortOrder;
     }
-    
+
     public void setGlobalSort(boolean globalSort) {
         this.globalSort = globalSort;
     }
-    
+
     public boolean isGlobalSort() {
         return globalSort;
     }
-    
+
     public boolean isLimitAfterSort() {
         return limitAfterSort;
     }

Added: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java (added)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java Tue Dec  3 18:07:50 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.executionengine.tez;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.tez.runtime.api.LogicalOutput;
+
+/**
+ * This interface is implemented by PhysicalOperators that can have Tez outputs
+ * attached directly to the operator.
+ */
+
+public interface TezOutput {
+    public void attachOutputs(Map<String, LogicalOutput> outputs,
+            Configuration conf) throws ExecException;
+}

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java Tue Dec  3 18:07:50 2013
@@ -60,13 +60,13 @@ public class TezPOPackageAnnotator exten
         }
     }
 
-    private void handlePackage(TezOperator tezOp, POPackage pkg) throws VisitorException {
+    private void handlePackage(TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
         // the LocalRearrange(s) must be in the plan of a predecessor tez op
         int lrFound = 0;
-            List<TezOperator> preds = this.mPlan.getPredecessors(tezOp);
+            List<TezOperator> preds = this.mPlan.getPredecessors(pkgTezOp);
             for (Iterator<TezOperator> it = preds.iterator(); it.hasNext();) {
-                TezOperator tezOper = it.next();
-                lrFound += patchPackage(tezOper.plan, pkg);
+                TezOperator predTezOp = it.next();
+                lrFound += patchPackage(predTezOp, pkgTezOp, pkg);
                 if(lrFound == pkg.getNumInps()) {
                     break;
                 }
@@ -79,8 +79,8 @@ public class TezPOPackageAnnotator exten
         }
     }
 
-    private int patchPackage(PhysicalPlan plan, POPackage pkg) throws VisitorException {
-        LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(plan, pkg);
+    private int patchPackage(TezOperator predTezOp, TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
+        LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(predTezOp.plan, pkgTezOp, pkg);
         lrDiscoverer.visit();
         // let our caller know if we managed to patch
         // the package
@@ -124,15 +124,21 @@ public class TezPOPackageAnnotator exten
     static class LoRearrangeDiscoverer extends PhyPlanVisitor {
 
         private int loRearrangeFound = 0;
+        private TezOperator pkgTezOp;
         private POPackage pkg;
 
-        public LoRearrangeDiscoverer(PhysicalPlan plan, POPackage pkg) {
+        public LoRearrangeDiscoverer(PhysicalPlan plan, TezOperator pkgTezOp, POPackage pkg) {
             super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
+            this.pkgTezOp = pkgTezOp;
             this.pkg = pkg;
         }
 
         @Override
         public void visitLocalRearrange(POLocalRearrange lrearrange) throws VisitorException {
+            POLocalRearrangeTez lr = (POLocalRearrangeTez) lrearrange;
+            if (!lr.getOutputKey().equals(pkgTezOp.getOperatorKey().toString())) {
+                return;
+            }
             loRearrangeFound++;
             Map<Integer,Pair<Boolean, Map<Integer, Integer>>> keyInfo;
 

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java Tue Dec  3 18:07:50 2013
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.pig.impl.PigContext;
@@ -75,7 +76,7 @@ public class TezSessionManager {
         }
     }
 
-    private static SessionInfo createSession(Configuration conf, Map<String, LocalResource> requestedAMResources) throws TezException, IOException {
+    private static SessionInfo createSession(Configuration conf, Map<String, LocalResource> requestedAMResources, Credentials creds) throws TezException, IOException {
         TezConfiguration tezConf = new TezConfiguration(conf);
         TezClient tezClient = new TezClient(tezConf);
         ApplicationId appId = tezClient.createApplication();
@@ -84,7 +85,7 @@ public class TezSessionManager {
         resources.putAll(requestedAMResources);
 
         String jobName = conf.get(PigContext.JOB_NAME, "pig");
-        AMConfiguration amConfig = new AMConfiguration(null, resources, tezConf, null);
+        AMConfiguration amConfig = new AMConfiguration(null, resources, tezConf, creds);
         TezSessionConfiguration sessionConfig = new TezSessionConfiguration(amConfig, tezConf);
         TezSession tezSession = new TezSession(jobName, appId, sessionConfig);
         tezSession.start();
@@ -101,7 +102,7 @@ public class TezSessionManager {
         return true;
     }
 
-    static TezSession getSession(Configuration conf, Map<String, LocalResource> requestedAMResources) throws TezException, IOException {
+    static TezSession getSession(Configuration conf, Map<String, LocalResource> requestedAMResources, Credentials creds) throws TezException, IOException {
         synchronized (sessionPool) {
             List<SessionInfo> sessionsToRemove = new ArrayList<SessionInfo>();
             for (SessionInfo sessionInfo : sessionPool) {
@@ -120,7 +121,7 @@ public class TezSessionManager {
             }
 
             // We cannot find available AM, create new one
-            SessionInfo sessionInfo = createSession(conf, requestedAMResources);
+            SessionInfo sessionInfo = createSession(conf, requestedAMResources, creds);
             sessionInfo.inUse = true;
             sessionPool.add(sessionInfo);
             return sessionInfo.session;

Modified: pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java Tue Dec  3 18:07:50 2013
@@ -74,7 +74,7 @@ public class TezStats extends PigStats {
                     }
                 }
             }
-            tezOpVertexMap.put(tezOp.name(), currStats);
+            tezOpVertexMap.put(tezOp.getOperatorKey().toString(), currStats);
         }
     }
 

Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld Tue Dec  3 18:07:50 2013
@@ -0,0 +1,51 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-148
+#--------------------------------------------------
+Tez vertex scope-142
+# Plan on vertex
+6-1: Split - scope-126
+|   |
+|   b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-131
+|   |
+|   |---b: Filter[bag] - scope-127
+|       |   |
+|       |   Less Than or Equal[boolean] - scope-130
+|       |   |
+|       |   |---Project[int][0] - scope-128
+|       |   |
+|       |   |---Constant(5) - scope-129
+|   |
+|   c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-136
+|   |
+|   |---c: Filter[bag] - scope-132
+|       |   |
+|       |   Less Than or Equal[boolean] - scope-135
+|       |   |
+|       |   |---Project[int][0] - scope-133
+|       |   |
+|       |   |---Constant(10) - scope-134
+|   |
+|   d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-141
+|   |
+|   |---d: Filter[bag] - scope-137
+|       |   |
+|       |   Greater Than[boolean] - scope-140
+|       |   |
+|       |   |---Project[int][0] - scope-138
+|       |   |
+|       |   |---Constant(10) - scope-139
+|
+|---a: New For Each(false,false)[bag] - scope-125
+    |   |
+    |   Cast[int] - scope-120
+    |   |
+    |   |---Project[bytearray][0] - scope-119
+    |   |
+    |   Cast[int] - scope-123
+    |   |
+    |   |---Project[bytearray][1] - scope-122
+    |
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-118
\ No newline at end of file

Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld Tue Dec  3 18:07:50 2013
@@ -0,0 +1,81 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-196
+#--------------------------------------------------
+Tez vertex scope-182
+# Plan on vertex
+7-1: Split - scope-157
+|   |
+|   b1: Local Rearrange[tuple]{int}(false) - scope-164
+|   |   |
+|   |   Project[int][0] - scope-165
+|   |
+|   |---b: Filter[bag] - scope-158
+|       |   |
+|       |   Less Than or Equal[boolean] - scope-161
+|       |   |
+|       |   |---Project[int][0] - scope-159
+|       |   |
+|       |   |---Constant(5) - scope-160
+|   |
+|   Local Rearrange[tuple]{tuple}(false) - scope-188
+|   |   |
+|   |   Project[tuple][*] - scope-187
+|   |
+|   |---c1: Limit - scope-171
+|       |
+|       |---c: Filter[bag] - scope-167
+|           |   |
+|           |   Less Than or Equal[boolean] - scope-170
+|           |   |
+|           |   |---Project[int][0] - scope-168
+|           |   |
+|           |   |---Constant(10) - scope-169
+|   |
+|   d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-181
+|   |
+|   |---d1: Filter[bag] - scope-177
+|       |   |
+|       |   Equal To[boolean] - scope-180
+|       |   |
+|       |   |---Project[int][0] - scope-178
+|       |   |
+|       |   |---Constant(5) - scope-179
+|       |
+|       |---d: Filter[bag] - scope-173
+|           |   |
+|           |   Greater Than[boolean] - scope-176
+|           |   |
+|           |   |---Project[int][0] - scope-174
+|           |   |
+|           |   |---Constant(10) - scope-175
+|
+|---a: New For Each(false,false)[bag] - scope-156
+    |   |
+    |   Cast[int] - scope-151
+    |   |
+    |   |---Project[bytearray][0] - scope-150
+    |   |
+    |   Cast[int] - scope-154
+    |   |
+    |   |---Project[bytearray][1] - scope-153
+    |
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-149
+Tez vertex scope-184
+# Plan on vertex
+b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-166
+|
+|---b1: Package(Packager)[tuple]{int} - scope-163
+Tez vertex scope-189
+# Plan on vertex
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-172
+|
+|---Limit - scope-193
+    |
+    |---New For Each(true)[bag] - scope-192
+        |   |
+        |   Project[tuple][1] - scope-191
+        |
+        |---Package(Packager)[tuple]{tuple} - scope-190
\ No newline at end of file

Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld Tue Dec  3 18:07:50 2013
@@ -0,0 +1,49 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-227
+#--------------------------------------------------
+Tez vertex scope-221
+# Plan on vertex
+a: Split - scope-205
+|   |
+|   b: Local Rearrange[tuple]{int}(false) - scope-210
+|   |   |
+|   |   Project[int][0] - scope-211
+|   |
+|   |---a: Filter[bag] - scope-206
+|       |   |
+|       |   Constant(true) - scope-207
+|   |
+|   c: Local Rearrange[tuple]{tuple}(false) - scope-217
+|   |   |
+|   |   Project[int][0] - scope-218
+|   |   |
+|   |   Project[int][1] - scope-219
+|   |
+|   |---a: Filter[bag] - scope-213
+|       |   |
+|       |   Constant(true) - scope-214
+|
+|---a: New For Each(false,false)[bag] - scope-204
+    |   |
+    |   Cast[int] - scope-199
+    |   |
+    |   |---Project[bytearray][0] - scope-198
+    |   |
+    |   Cast[int] - scope-202
+    |   |
+    |   |---Project[bytearray][1] - scope-201
+    |
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-197
+Tez vertex scope-226
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-220
+|
+|---c: Package(Packager)[tuple]{tuple} - scope-216
+Tez vertex scope-223
+# Plan on vertex
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-212
+|
+|---b: Package(Packager)[tuple]{int} - scope-209
\ No newline at end of file

Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java Tue Dec  3 18:07:50 2013
@@ -53,11 +53,14 @@ import org.junit.runner.RunWith;
  */
 @RunWith(OrderedJUnit4Runner.class)
 @TestOrder({
-    "testRun1",
-    "testRun2",
-    "testRun3",
-    "testRun4",
-    "testRun5"
+    "testFilter",
+    "testGroupBy",
+    "testJoin",
+    "testLimit",
+    "testDistinct",
+    "testSplitSingleVertex",
+    "testSplitMultiVertex",
+    "testMultipleGroupBySplit"
 })
 public class TestTezCompiler {
     private static PigContext pc;
@@ -79,7 +82,7 @@ public class TestTezCompiler {
     }
 
     @Test
-    public void testRun1() throws Exception {
+    public void testFilter() throws Exception {
         String query =
                 "a = load 'file:///tmp/input' as (x:int, y:int);" +
                 "b = filter a by x > 0;" +
@@ -91,7 +94,7 @@ public class TestTezCompiler {
     }
 
     @Test
-    public void testRun2() throws Exception {
+    public void testGroupBy() throws Exception {
         String query =
                 "a = load 'file:///tmp/input' as (x:int, y:int);" +
                 "b = group a by x;" +
@@ -103,7 +106,7 @@ public class TestTezCompiler {
     }
 
     @Test
-    public void testRun3() throws Exception {
+    public void testJoin() throws Exception {
         String query =
                 "a = load 'file:///tmp/input1' as (x:int, y:int);" +
                 "b = load 'file:///tmp/input2' as (x:int, z:int);" +
@@ -116,7 +119,7 @@ public class TestTezCompiler {
     }
 
     @Test
-    public void testRun4() throws Exception {
+    public void testLimit() throws Exception {
         String query =
                 "a = load 'file:///tmp/input' as (x:int, y:int);" +
                 "b = limit a 10;" +
@@ -128,7 +131,7 @@ public class TestTezCompiler {
     }
 
     @Test
-    public void testRun5() throws Exception {
+    public void testDistinct() throws Exception {
         String query =
                 "a = load 'file:///tmp/input' as (x:int, y:int);" +
                 "b = distinct a;" +
@@ -139,6 +142,52 @@ public class TestTezCompiler {
         run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld");
     }
 
+    @Test
+    public void testSplitSingleVertex() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+                "store b into 'file:///tmp/output/b';" +
+                "store c into 'file:///tmp/output/c';" +
+                "store d into 'file:///tmp/output/d';";
+
+        PhysicalPlan pp = Util.buildPp(pigServer, query);
+        run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld");
+    }
+
+    @Test
+    public void testSplitMultiVertex() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+                "b1 = group b by x;" +
+                "c1 = limit c 1;" +
+                "d1 = filter d by x == 5;" +
+                //"d2 = order d by x;" + //TODO
+                //"d3 = join c1 by x, d1 by x;" +
+                "store b1 into 'file:///tmp/output/b1';" +
+                "store c1 into 'file:///tmp/output/c1';" +
+                "store d1 into 'file:///tmp/output/d1';";
+                //"store d2 into 'file:///tmp/output/d2';" +
+                //"store d3 into 'file:///tmp/output/d3';";
+
+        PhysicalPlan pp = Util.buildPp(pigServer, query);
+        run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld");
+    }
+
+    @Test
+    public void testMultipleGroupBySplit() throws Exception {
+        String query =
+                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "b = group a by x;" +
+                "c = group a by (x,y);" +
+                "store b into 'file:///tmp/output/b';" +
+                "store c into 'file:///tmp/output/c';";
+
+        PhysicalPlan pp = Util.buildPp(pigServer, query);
+        run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld");
+    }
+
     private void run(PhysicalPlan pp, String expectedFile) throws Exception {
         TezCompiler comp = new TezCompiler(pp, pc);
         comp.compile();
@@ -149,6 +198,8 @@ public class TestTezCompiler {
         TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, tezPlanContainer);
         printer.visit();
         String compiledPlan = baos.toString();
+        System.out.println();
+        System.out.println("<<<" + compiledPlan + ">>>");
 
         FileInputStream fis = new FileInputStream(expectedFile);
         byte[] b = new byte[MAX_SIZE];
@@ -159,8 +210,6 @@ public class TestTezCompiler {
             goldenPlan = goldenPlan.substring(0, len-1);
         }
 
-        System.out.println();
-        System.out.println("<<<" + compiledPlan + ">>>");
         System.out.println("-------------");
         System.out.println("Golden");
         System.out.println("<<<" + goldenPlan + ">>>");

Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java Tue Dec  3 18:07:50 2013
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+import java.net.URI;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Properties;
@@ -60,10 +61,14 @@ import org.junit.runner.RunWith;
 public class TestTezJobControlCompiler {
     private static PigContext pc;
     private static PigServer pigServer;
+    private static URI input1;
+    private static URI input2;
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
         pc = new PigContext(new TezLocalExecType(), new Properties());
+        input1 = Util.createTempFileDelOnExit("input1", "txt").toURI();
+        input2 = Util.createTempFileDelOnExit("input2", "txt").toURI();
     }
 
     @AfterClass
@@ -78,7 +83,7 @@ public class TestTezJobControlCompiler {
     @Test
     public void testRun1() throws Exception {
         String query =
-                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "a = load '" + input1 +"' as (x:int, y:int);" +
                 "b = filter a by x > 0;" +
                 "c = foreach b generate y;" +
                 "store c into 'file:///tmp/output';";
@@ -89,7 +94,7 @@ public class TestTezJobControlCompiler {
         List<TezOperator> roots = compiledPlan.first.getRoots();
         assertEquals(1, roots.size());
 
-        Vertex root = compiledPlan.second.getVertex(roots.get(0).name());
+        Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
         assertNotNull(root);
         assertEquals(0, root.getInputVertices().size());
         assertEquals(0, root.getOutputVertices().size());
@@ -98,7 +103,7 @@ public class TestTezJobControlCompiler {
     @Test
     public void testRun2() throws Exception {
         String query =
-                "a = load 'file:///tmp/input' as (x:int, y:int);" +
+                "a = load '" + input1 +"' as (x:int, y:int);" +
                 "b = group a by x;" +
                 "c = foreach b generate group, a;" +
                 "store c into 'file:///tmp/output';";
@@ -112,12 +117,12 @@ public class TestTezJobControlCompiler {
         List<TezOperator> leaves = compiledPlan.first.getLeaves();
         assertEquals(1, leaves.size());
 
-        Vertex root = compiledPlan.second.getVertex(roots.get(0).name());
+        Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
         assertNotNull(root);
         assertEquals(0, root.getInputVertices().size());
         assertEquals(1, root.getOutputVertices().size());
 
-        Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).name());
+        Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
         assertNotNull(leaf);
         assertEquals(1, leaf.getInputVertices().size());
         assertEquals(0, leaf.getOutputVertices().size());
@@ -129,8 +134,8 @@ public class TestTezJobControlCompiler {
     @Test
     public void testRun3() throws Exception {
         String query =
-                "a = load 'file:///tmp/input1' as (x:int, y:int);" +
-                "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+                "a = load '" + input1 +"' as (x:int, y:int);" +
+                "b = load '" + input2 +"' as (x:int, z:int);" +
                 "c = join a by x, b by x;" +
                 "d = foreach c generate a::x as x, y, z;" +
                 "store d into 'file:///tmp/output';";
@@ -144,14 +149,14 @@ public class TestTezJobControlCompiler {
         List<TezOperator> leaves = compiledPlan.first.getLeaves();
         assertEquals(1, leaves.size());
 
-        Vertex root0 = compiledPlan.second.getVertex(roots.get(0).name());
-        Vertex root1 = compiledPlan.second.getVertex(roots.get(1).name());
+        Vertex root0 = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
+        Vertex root1 = compiledPlan.second.getVertex(roots.get(1).getOperatorKey().toString());
         assertNotNull(root0);
         assertNotNull(root1);
         assertEquals(0, root0.getInputVertices().size());
         assertEquals(1, root1.getOutputVertices().size());
 
-        Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).name());
+        Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
         assertNotNull(leaf);
         assertEquals(2, leaf.getInputVertices().size());
         assertEquals(0, leaf.getOutputVertices().size());