You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/08/13 21:48:20 UTC
svn commit: r1617821 [1/2] - in /pig/branches/maven: ./
pig/shims/test/hadoop23/org/apache/pig/test/
pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/
pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/
pi...
Author: cheolsoo
Date: Wed Aug 13 19:48:19 2014
New Revision: 1617821
URL: http://svn.apache.org/r1617821
Log:
Merge latest trunk changes
Added:
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/NativeTezOper.java
Modified:
pig/branches/maven/ (props changed)
pig/branches/maven/CHANGES.txt
pig/branches/maven/pig/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java
pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
pig/branches/maven/pig/src/main/java/org/apache/pig/impl/plan/OperatorPlan.java
pig/branches/maven/pig/src/main/java/org/apache/pig/parser/QueryLexer.g
pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezScriptState.java
pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezStats.java
pig/branches/maven/pig/test/e2e/pig/tests/nightly.conf
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
pig/branches/maven/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2 (props changed)
Propchange: pig/branches/maven/
------------------------------------------------------------------------------
Merged /pig/trunk:r1617784-1617805
Modified: pig/branches/maven/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/maven/CHANGES.txt?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/CHANGES.txt (original)
+++ pig/branches/maven/CHANGES.txt Wed Aug 13 19:48:19 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4114: Add Native operator to tez (daijy)
+
PIG-4117: Implement merge cogroup in Tez (daijy)
PIG-4119: Add message at end of each testcase with timestamp in Pig system tests (nmaheshwari via daijy)
@@ -60,6 +62,10 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4123: Increase memory for TezMiniCluster (daijy)
+
+PIG-4122: Fix hadoopversion 23 compilation due to TEZ-1194 (daijy)
+
PIG-4061: Make Streaming UDF work in Tez (daijy)
PIG-4118: Fix hadoopversion 23 compilation due to TEZ-1237/TEZ-1407 (daijy)
Modified: pig/branches/maven/pig/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java (original)
+++ pig/branches/maven/pig/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java Wed Aug 13 19:48:19 2014
@@ -37,6 +37,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
import org.apache.pig.backend.hadoop.executionengine.tez.TezSessionManager;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
public class TezMiniCluster extends MiniGenericCluster {
@@ -99,6 +100,8 @@ public class TezMiniCluster extends Mini
m_mr_conf.set(MRConfiguration.FRAMEWORK_NAME, "yarn-tez");
m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
System.getProperty("java.class.path"));
+ m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx2048m");
+ m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2048m");
Configuration mapred_site = new Configuration(false);
Configuration yarn_site = new Configuration(false);
Added: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/NativeTezOper.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/NativeTezOper.java?rev=1617821&view=auto
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/NativeTezOper.java (added)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/NativeTezOper.java Wed Aug 13 19:48:19 2014
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.backend.hadoop.executionengine.tez;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.util.RunJar;
+import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.RunJarSecurityManager;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.tez.TezStats;
+
+public class NativeTezOper extends TezOperator {
+ private static final long serialVersionUID = 1L;
+ private static int countJobs = 0;
+ private String nativeTezJar;
+ private String[] params;
+ public NativeTezOper(OperatorKey k, String tezJar, String[] parameters) {
+ super(k);
+ nativeTezJar = tezJar;
+ params = parameters;
+ }
+
+ public static int getJobNumber() {
+ countJobs++;
+ return countJobs;
+ }
+
+ public String getJobId() {
+ return nativeTezJar + "_";
+ }
+
+ public String getCommandString() {
+ StringBuilder sb = new StringBuilder("hadoop jar ");
+ sb.append(nativeTezJar);
+ for(String pr: params) {
+ sb.append(" ");
+ sb.append(pr);
+ }
+ return sb.toString();
+ }
+
+ private String[] getNativeTezParams() {
+ // may need more cooking
+ String[] paramArr = new String[params.length + 1];
+ paramArr[0] = nativeTezJar;
+ for(int i=0; i< params.length; i++) {
+ paramArr[i+1] = params[i];
+ }
+ return paramArr;
+ }
+
+ @Override
+ public void visit(TezOpPlanVisitor v) throws VisitorException {
+ v.visitTezOp(this);
+ }
+
+ public void runJob() throws JobCreationException {
+ RunJarSecurityManager secMan = new RunJarSecurityManager();
+ try {
+ RunJar.main(getNativeTezParams());
+ ((TezStats)PigStats.get()).addTezJobStatsForNative(this, true);
+ } catch (SecurityException se) {
+ if(secMan.getExitInvoked()) {
+ if(secMan.getExitCode() != 0) {
+ throw new JobCreationException("Native job returned with non-zero return code");
+ }
+ else {
+ ((TezStats)PigStats.get()).addTezJobStatsForNative(this, true);
+ }
+ }
+ } catch (Throwable t) {
+ JobCreationException e = new JobCreationException(
+ "Cannot run native tez job "+ t.getMessage(), t);
+ ((TezStats)PigStats.get()).addTezJobStatsForNative(this, false);
+ throw e;
+ } finally {
+ secMan.retire();
+ }
+ }
+
+ @Override
+ public String name(){
+ return "Tez - " + mKey.toString() + "\n"
+ + " Native Tez - jar : " + nativeTezJar + ", params: " + Arrays.toString(params) ;
+
+
+ }
+
+}
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java Wed Aug 13 19:48:19 2014
@@ -52,6 +52,7 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.tools.pigstats.PigStatusReporter;
import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
import org.apache.tez.runtime.api.Event;
@@ -113,7 +114,7 @@ public class PigProcessor extends Abstra
PhysicalOperator.reporter = new ThreadLocal<PigProgressable>();
PigMapReduce.sJobConfInternal = new ThreadLocal<Configuration>();
- byte[] payload = getContext().getUserPayload();
+ UserPayload payload = getContext().getUserPayload();
conf = TezUtils.createConfFromUserPayload(payload);
PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
.deserialize(conf.get("udf.import.list")));
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java Wed Aug 13 19:48:19 2014
@@ -226,8 +226,16 @@ public class TezCompiler extends PhyPlan
// get all stores, sort them in order(operator id) and compile their
// plans
List<POStore> stores = PlanHelper.getPhysicalOperators(plan, POStore.class);
- List<PhysicalOperator> ops = Lists.newArrayList();
- ops.addAll(stores);
+ List<PONative> nativeTezs= PlanHelper.getPhysicalOperators(plan, PONative.class);
+ List<PhysicalOperator> ops;
+ if (!pigContext.inIllustrator) {
+ ops = new ArrayList<PhysicalOperator>(stores.size() + nativeTezs.size());
+ ops.addAll(stores);
+ } else {
+ ops = new ArrayList<PhysicalOperator>(leaves.size() + nativeTezs.size());
+ ops.addAll(leaves);
+ }
+ ops.addAll(nativeTezs);
Collections.sort(ops);
for (PhysicalOperator op : ops) {
@@ -298,7 +306,11 @@ public class TezCompiler extends PhyPlan
// Compile each predecessor into the TezOperator and store them away so
// that we can use them for compiling op.
List<PhysicalOperator> predecessors = plan.getPredecessors(op);
- if (predecessors != null && predecessors.size() > 0) {
+ if(op instanceof PONative){
+ // the predecessor (store) has already been processed
+ // don't process it again
+ }
+ else if (predecessors != null && predecessors.size() > 0) {
// When processing an entire script (multiquery), we can get into a
// situation where a load has predecessors. This means that it
// depends on some store earlier in the plan. We need to take that
@@ -313,49 +325,60 @@ public class TezCompiler extends PhyPlan
}
PhysicalOperator p = predecessors.get(0);
- TezOperator storeTezOper = null;
- if (p instanceof POStore) {
- storeTezOper = phyToTezOpMap.get(p);
+ TezOperator oper = null;
+ if (p instanceof POStore || p instanceof PONative) {
+ oper = phyToTezOpMap.get(p);
} else {
int errCode = 2126;
- String msg = "Predecessor of load should be a store. Got " + p.getClass();
+ String msg = "Predecessor of load should be a store or native oper. Got " + p.getClass();
throw new PlanException(msg, errCode, PigException.BUG);
}
- PhysicalOperator store = storeTezOper.plan.getOperator(p.getOperatorKey());
- // replace POStore to POValueOutputTez, convert the tezOperator to splitter
- storeTezOper.plan.disconnect(storeTezOper.plan.getPredecessors(store).get(0), store);
- storeTezOper.plan.remove(store);
- POValueOutputTez valueOutput = new POValueOutputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
- storeTezOper.plan.addAsLeaf(valueOutput);
- storeTezOper.setSplitter(true);
-
- // Create a splittee of store only
- TezOperator storeOnlyTezOperator = getTezOp();
- PhysicalPlan storeOnlyPhyPlan = new PhysicalPlan();
- POValueInputTez valueInput = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
- valueInput.setInputKey(storeTezOper.getOperatorKey().toString());
- storeOnlyPhyPlan.addAsLeaf(valueInput);
- storeOnlyPhyPlan.addAsLeaf(store);
- storeOnlyTezOperator.plan = storeOnlyPhyPlan;
- tezPlan.add(storeOnlyTezOperator);
- phyToTezOpMap.put(store, storeOnlyTezOperator);
-
- // Create new operator as second splittee
- curTezOp = getTezOp();
- POValueInputTez valueInput2 = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
- valueInput2.setInputKey(storeTezOper.getOperatorKey().toString());
- curTezOp.plan.add(valueInput2);
- tezPlan.add(curTezOp);
-
- // Connect splitter to splittee
- TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, storeTezOper, storeOnlyTezOperator);
- TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
- storeOnlyTezOperator.setRequestedParallelismByReference(storeTezOper);
-
- edge = TezCompilerUtil.connect(tezPlan, storeTezOper, curTezOp);
- TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
- curTezOp.setRequestedParallelismByReference(storeTezOper);
-
+ if (p instanceof POStore) {
+ PhysicalOperator store = oper.plan.getOperator(p.getOperatorKey());
+ // replace POStore to POValueOutputTez, convert the tezOperator to splitter
+ oper.plan.disconnect(oper.plan.getPredecessors(store).get(0), store);
+ oper.plan.remove(store);
+ POValueOutputTez valueOutput = new POValueOutputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
+ oper.plan.addAsLeaf(valueOutput);
+ oper.setSplitter(true);
+
+ // Create a splittee of store only
+ TezOperator storeOnlyTezOperator = getTezOp();
+ PhysicalPlan storeOnlyPhyPlan = new PhysicalPlan();
+ POValueInputTez valueInput = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
+ valueInput.setInputKey(oper.getOperatorKey().toString());
+ storeOnlyPhyPlan.addAsLeaf(valueInput);
+ storeOnlyPhyPlan.addAsLeaf(store);
+ storeOnlyTezOperator.plan = storeOnlyPhyPlan;
+ tezPlan.add(storeOnlyTezOperator);
+ phyToTezOpMap.put(store, storeOnlyTezOperator);
+
+ // Create new operator as second splittee
+ curTezOp = getTezOp();
+ POValueInputTez valueInput2 = new POValueInputTez(new OperatorKey(scope,nig.getNextNodeId(scope)));
+ valueInput2.setInputKey(oper.getOperatorKey().toString());
+ curTezOp.plan.add(valueInput2);
+ tezPlan.add(curTezOp);
+
+ // Connect splitter to splittee
+ TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, oper, storeOnlyTezOperator);
+ TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
+ storeOnlyTezOperator.setRequestedParallelismByReference(oper);
+
+ edge = TezCompilerUtil.connect(tezPlan, oper, curTezOp);
+ TezCompilerUtil.configureValueOnlyTupleOutput(edge, DataMovementType.ONE_TO_ONE);
+ curTezOp.setRequestedParallelismByReference(oper);
+ } else if (p instanceof PONative) {
+ // Need new operator
+ curTezOp = getTezOp();
+ curTezOp.plan.add(op);
+ tezPlan.add(curTezOp);
+
+ plan.disconnect(op, p);
+ TezCompilerUtil.connect(tezPlan, oper, curTezOp);
+ phyToTezOpMap.put(op, curTezOp);
+ return;
+ }
return;
}
@@ -1273,9 +1296,22 @@ public class TezCompiler extends PhyPlan
@Override
public void visitNative(PONative op) throws VisitorException {
- int errCode = 2034;
- String msg = "Cannot compile " + op.getClass().getSimpleName();
- throw new TezCompilerException(msg, errCode, PigException.BUG);
+ try{
+ curTezOp.segmentBelow = true;
+ TezOperator nativeTezOper = new NativeTezOper(new OperatorKey(scope,nig.getNextNodeId(scope)), op.getNativeMRjar(), op.getParams());
+ tezPlan.add(nativeTezOper);
+ curTezOp.setClosed(true);
+ tezPlan.connect(curTezOp, nativeTezOper);
+ phyToTezOpMap.put(op, nativeTezOper);
+ nativeTezOper.setClosed(true);
+ nativeTezOper.segmentBelow = true;
+ nativeTezOper.markNative();
+ curTezOp = nativeTezOper;
+ }catch(Exception e){
+ int errCode = 2034;
+ String msg = "Error compiling operator " + op.getClass().getSimpleName();
+ throw new TezCompilerException(msg, errCode, PigException.BUG, e);
+ }
}
@Override
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Wed Aug 13 19:48:19 2014
@@ -116,6 +116,7 @@ import org.apache.tez.dag.api.InputIniti
import org.apache.tez.dag.api.OutputCommitterDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.VertexLocationHint;
@@ -559,7 +560,7 @@ public class TezDagBuilder extends TezOp
}
// Take our assembled configuration and create a vertex
- byte[] userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf);
procDesc.setUserPayload(userPayload);
Vertex vertex = new Vertex(tezOp.getOperatorKey().toString(), procDesc, tezOp.getVertexParallelism(),
@@ -597,9 +598,9 @@ public class TezDagBuilder extends TezOp
vertex.setLocationHint(new VertexLocationHint(tezOp.getLoaderInfo().getInputSplitInfo().getTaskLocationHints()));
vertex.addDataSource(ld.getOperatorKey().toString(),
new DataSourceDescriptor(new InputDescriptor(MRInput.class.getName())
- .setUserPayload(MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
+ .setUserPayload(new UserPayload(MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
.setConfigurationBytes(TezUtils.createByteStringFromConf(payloadConf))
- .setSplits(tezOp.getLoaderInfo().getInputSplitInfo().getSplitsProto()).build().toByteArray()),
+ .setSplits(tezOp.getLoaderInfo().getInputSplitInfo().getSplitsProto()).build().toByteArray())),
new InputInitializerDescriptor(MRInputSplitDistributor.class.getName()), dag.getCredentials()));
}
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Wed Aug 13 19:48:19 2014
@@ -48,6 +48,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
+import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.UDFContext;
@@ -58,6 +59,7 @@ import org.apache.pig.tools.pigstats.tez
import org.apache.pig.tools.pigstats.tez.TezTaskStats;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
@@ -112,54 +114,64 @@ public class TezLauncher extends Launche
TezOperPlan tezPlan;
while ((tezPlan=tezPlanContainer.getNextPlan(processedPlans)) != null) {
+ optimize(tezPlan, pc);
processedPlans.add(tezPlan);
-
- TezPOPackageAnnotator pkgAnnotator = new TezPOPackageAnnotator(tezPlan);
- pkgAnnotator.visit();
-
+ ProgressReporter reporter = new ProgressReporter();
tezStats.initialize(tezPlan);
- runningJob = jc.compile(tezPlan, grpName, tezPlanContainer);
-
- tezScriptState.emitInitialPlanNotification(tezPlan);
- tezScriptState.emitLaunchStartedNotification(tezPlan.size());
-
- // Set the thread UDFContext so registered classes are available.
- final UDFContext udfContext = UDFContext.getUDFContext();
- Thread task = new Thread(runningJob) {
- @Override
- public void run() {
- UDFContext.setUdfContext(udfContext.clone());
- super.run();
+ if (tezPlan.size()==1 && tezPlan.getRoots().get(0) instanceof NativeTezOper) {
+ // Native Tez Plan
+ NativeTezOper nativeOper = (NativeTezOper)tezPlan.getRoots().get(0);
+ tezScriptState.emitInitialPlanNotification(tezPlan);
+ tezScriptState.emitLaunchStartedNotification(tezPlan.size());
+ tezScriptState.emitJobsSubmittedNotification(1);
+ nativeOper.runJob();
+ } else {
+ TezPOPackageAnnotator pkgAnnotator = new TezPOPackageAnnotator(tezPlan);
+ pkgAnnotator.visit();
+
+ runningJob = jc.compile(tezPlan, grpName, tezPlanContainer);
+
+ tezScriptState.emitInitialPlanNotification(tezPlan);
+ tezScriptState.emitLaunchStartedNotification(tezPlan.size());
+
+ // Set the thread UDFContext so registered classes are available.
+ final UDFContext udfContext = UDFContext.getUDFContext();
+ Thread task = new Thread(runningJob) {
+ @Override
+ public void run() {
+ UDFContext.setUdfContext(udfContext.clone());
+ super.run();
+ }
+ };
+
+ JobControlThreadExceptionHandler jctExceptionHandler = new JobControlThreadExceptionHandler();
+ task.setUncaughtExceptionHandler(jctExceptionHandler);
+ task.setContextClassLoader(PigContext.getClassLoader());
+
+ tezStats.setTezJob(runningJob);
+
+ // Mark the times that the jobs were submitted so it's reflected in job
+ // history props
+ long scriptSubmittedTimestamp = System.currentTimeMillis();
+ // Job.getConfiguration returns the shared configuration object
+ Configuration jobConf = runningJob.getConfiguration();
+ jobConf.set("pig.script.submitted.timestamp",
+ Long.toString(scriptSubmittedTimestamp));
+ jobConf.set("pig.job.submitted.timestamp",
+ Long.toString(System.currentTimeMillis()));
+
+ Future<?> future = executor.schedule(task, timeToSleep, TimeUnit.MILLISECONDS);
+
+ tezScriptState.emitJobsSubmittedNotification(1);
+ reporter.notifyStarted();
+
+ while (!future.isDone()) {
+ reporter.notifyUpdate();
+ Thread.sleep(1000);
}
- };
-
- JobControlThreadExceptionHandler jctExceptionHandler = new JobControlThreadExceptionHandler();
- task.setUncaughtExceptionHandler(jctExceptionHandler);
- task.setContextClassLoader(PigContext.getClassLoader());
-
- tezStats.setTezJob(runningJob);
-
- // Mark the times that the jobs were submitted so it's reflected in job
- // history props
- long scriptSubmittedTimestamp = System.currentTimeMillis();
- // Job.getConfiguration returns the shared configuration object
- Configuration jobConf = runningJob.getConfiguration();
- jobConf.set("pig.script.submitted.timestamp",
- Long.toString(scriptSubmittedTimestamp));
- jobConf.set("pig.job.submitted.timestamp",
- Long.toString(System.currentTimeMillis()));
-
- Future<?> future = executor.schedule(task, timeToSleep, TimeUnit.MILLISECONDS);
- ProgressReporter reporter = new ProgressReporter();
- tezScriptState.emitJobsSubmittedNotification(1);
- reporter.notifyStarted();
-
- while (!future.isDone()) {
- reporter.notifyUpdate();
- Thread.sleep(1000);
+
+ tezStats.accumulateStats(runningJob);
}
-
- tezStats.accumulateStats(runningJob);
tezScriptState.emitProgressUpdatedNotification(100);
tezPlanContainer.updatePlan(tezPlan, reporter.notifyFinishedOrFailed());
}
@@ -217,8 +229,8 @@ public class TezLauncher extends Launche
public void notifyStarted() throws IOException {
for (Vertex v : runningJob.getDAG().getVertices()) {
TezTaskStats tts = tezStats.getVertexStats(v.getName());
- byte[] bb = v.getProcessorDescriptor().getUserPayload();
- Configuration conf = TezUtils.createConfFromUserPayload(bb);
+ UserPayload payload = v.getProcessorDescriptor().getUserPayload();
+ Configuration conf = TezUtils.createConfFromUserPayload(payload);
tts.setConf(conf);
tts.setId(v.getName());
tezScriptState.emitJobStartedNotification(v.getName());
@@ -269,6 +281,10 @@ public class TezLauncher extends Launche
VisitorException, IOException {
log.debug("Entering TezLauncher.explain");
TezPlanContainer tezPlanContainer = compile(php, pc);
+ for (Map.Entry<OperatorKey,TezPlanContainerNode> entry : tezPlanContainer.getKeys().entrySet()) {
+ TezOperPlan tezPlan = entry.getValue().getNode();
+ optimize(tezPlan, pc);
+ }
if (format.equals("text")) {
TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, tezPlanContainer);
@@ -280,11 +296,9 @@ public class TezLauncher extends Launche
}
}
- public TezPlanContainer compile(PhysicalPlan php, PigContext pc)
- throws PlanException, IOException, VisitorException {
+ public static void optimize(TezOperPlan tezPlan, PigContext pc) throws VisitorException {
Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
- TezCompiler comp = new TezCompiler(php, pc);
- TezOperPlan tezPlan = comp.compile();
+ boolean aggregateWarning = conf.getBoolean("aggregate.warning", false);
NoopFilterRemover filter = new NoopFilterRemover(tezPlan);
filter.visit();
@@ -336,7 +350,12 @@ public class TezLauncher extends Launche
ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
parallelismSetter.visit();
}
+ }
+ public TezPlanContainer compile(PhysicalPlan php, PigContext pc)
+ throws PlanException, IOException, VisitorException {
+ TezCompiler comp = new TezCompiler(php, pc);
+ comp.compile();
return comp.getPlanContainer();
}
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java Wed Aug 13 19:48:19 2014
@@ -150,7 +150,9 @@ public class TezOperator extends Operato
// Indicate if this job is a cogroup job
COGROUP,
// Indicate if this job is a regular join job
- HASHJOIN;
+ HASHJOIN,
+ // Indicate if this job is a native job
+ NATIVE;
};
OPER_FEATURE feature = OPER_FEATURE.NONE;
@@ -301,6 +303,14 @@ public class TezOperator extends Operato
feature = OPER_FEATURE.HASHJOIN;
}
+ public boolean isNative() {
+ return (feature == OPER_FEATURE.NATIVE);
+ }
+
+ public void markNative() {
+ feature = OPER_FEATURE.NATIVE;
+ }
+
public boolean isUnion() {
return union;
}
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/ParallelismSetter.java Wed Aug 13 19:48:19 2014
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigConfiguration;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.tez.NativeTezOper;
import org.apache.pig.backend.hadoop.executionengine.tez.TezEdgeDescriptor;
import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.tez.TezOperDependencyParallelismEstimator;
@@ -47,6 +48,9 @@ public class ParallelismSetter extends T
@Override
public void visitTezOp(TezOperator tezOp) throws VisitorException {
+ if (tezOp instanceof NativeTezOper) {
+ return;
+ }
try {
// Can only set parallelism here if the parallelism isn't derived from
// splits
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/backend/hadoop/executionengine/tez/util/TezCompilerUtil.java Wed Aug 13 19:48:19 2014
@@ -86,11 +86,13 @@ public class TezCompilerUtil {
static public TezEdgeDescriptor connect(TezOperPlan plan, TezOperator from, TezOperator to) throws PlanException {
plan.connect(from, to);
- PhysicalOperator leaf = from.plan.getLeaves().get(0);
- // It could be POStoreTez incase of sampling job in order by
- if (leaf instanceof POLocalRearrangeTez) {
- POLocalRearrangeTez lr = (POLocalRearrangeTez) leaf;
- lr.setOutputKey(to.getOperatorKey().toString());
+ if (!from.plan.isEmpty()) {
+ PhysicalOperator leaf = from.plan.getLeaves().get(0);
+ // It could be POStoreTez incase of sampling job in order by
+ if (leaf instanceof POLocalRearrangeTez) {
+ POLocalRearrangeTez lr = (POLocalRearrangeTez) leaf;
+ lr.setOutputKey(to.getOperatorKey().toString());
+ }
}
// Add edge descriptors to old and new operators
TezEdgeDescriptor edge = new TezEdgeDescriptor();
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/impl/plan/OperatorPlan.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/impl/plan/OperatorPlan.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/impl/plan/OperatorPlan.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/impl/plan/OperatorPlan.java Wed Aug 13 19:48:19 2014
@@ -21,8 +21,10 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Serializable;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -291,22 +293,32 @@ public abstract class OperatorPlan<E ext
* @throws PlanException
*/
public void moveTree(E root, OperatorPlan<E> newPlan) throws PlanException {
- newPlan.add(root);
- if (getSuccessors(root) == null) {
- remove(root);
- return;
+ Deque<E> queue = new ArrayDeque<E>();
+ queue.addLast(root);
+ while (!queue.isEmpty()) {
+ E node = queue.poll();
+ if (getSuccessors(node)!=null) {
+ for (E succ : getSuccessors(node)) {
+ if (!queue.contains(succ)) {
+ queue.addLast(succ);
+ }
+ }
+ }
+ newPlan.add(node);
}
-
- List<E> succs = new ArrayList<E>();
- succs.addAll(getSuccessors(root));
-
- for (E succ : succs) {
- moveTree(succ, newPlan);
+
+ for (E from : mFromEdges.keySet()) {
+ if (newPlan.mOps.containsKey(from)) {
+ for (E to : mFromEdges.get(from)) {
+ if (newPlan.mOps.containsKey(to)) {
+ newPlan.connect(from, to);
+ }
+ }
+ }
}
+
+ trimBelow(root);
remove(root);
- for (E succ : succs) {
- newPlan.connect(root, succ);
- }
}
/**
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/parser/QueryLexer.g?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/parser/QueryLexer.g Wed Aug 13 19:48:19 2014
@@ -234,7 +234,7 @@ STORE : 'STORE'
ASSERT : 'ASSERT'
;
-MAPREDUCE : 'MAPREDUCE'
+MAPREDUCE : 'MAPREDUCE' | 'NATIVE'
;
SHIP : 'SHIP'
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezScriptState.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezScriptState.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezScriptState.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezScriptState.java Wed Aug 13 19:48:19 2014
@@ -218,6 +218,9 @@ public class TezScriptState extends Scri
if (tezOp.isUnion()) {
feature.set(PIG_FEATURE.UNION.ordinal());
}
+ if (tezOp.isNative()) {
+ feature.set(PIG_FEATURE.NATIVE.ordinal());
+ }
try {
new FeatureVisitor(tezOp.plan, feature).visit();
} catch (VisitorException e) {
Modified: pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezStats.java
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezStats.java?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezStats.java (original)
+++ pig/branches/maven/pig/src/main/java/org/apache/pig/tools/pigstats/tez/TezStats.java Wed Aug 13 19:48:19 2014
@@ -28,7 +28,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobID;
import org.apache.pig.PigRunner.ReturnCode;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NativeMapReduceOper;
+import org.apache.pig.backend.hadoop.executionengine.tez.NativeTezOper;
import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
import org.apache.pig.backend.hadoop.executionengine.tez.TezJob;
import org.apache.pig.backend.hadoop.executionengine.tez.TezOpPlanVisitor;
@@ -41,6 +44,7 @@ import org.apache.pig.impl.plan.VisitorE
import org.apache.pig.tools.pigstats.InputStats;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
import org.apache.tez.common.TezUtils;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.DAGCounter;
@@ -49,6 +53,7 @@ import org.apache.tez.common.counters.Ta
import org.apache.tez.common.counters.TezCounter;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGStatus;
@@ -123,6 +128,13 @@ public class TezStats extends PigStats {
display();
}
+ public TezTaskStats addTezJobStatsForNative(NativeTezOper tezOper, boolean success) {
+ TezTaskStats js = tezOpVertexMap.get(tezOper.getOperatorKey().toString());
+ js.setId(tezOper.getJobId());
+ js.setSuccessful(success);
+ return js;
+ }
+
private void display() {
SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
StringBuilder sb = new StringBuilder();
@@ -204,8 +216,8 @@ public class TezStats extends PigStats {
for (String name : tezOpVertexMap.keySet()) {
Vertex v = dag.getVertex(name);
if (v != null) {
- byte[] bb = v.getProcessorDescriptor().getUserPayload();
- Configuration conf = TezUtils.createConfFromUserPayload(bb);
+ UserPayload payload = v.getProcessorDescriptor().getUserPayload();
+ Configuration conf = TezUtils.createConfFromUserPayload(payload);
addVertexStats(name, conf, succeeded, tezJob.getVertexCounters(name));
}
}
Modified: pig/branches/maven/pig/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/e2e/pig/tests/nightly.conf?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/e2e/pig/tests/nightly.conf (original)
+++ pig/branches/maven/pig/test/e2e/pig/tests/nightly.conf Wed Aug 13 19:48:19 2014
@@ -4012,7 +4012,7 @@ store e into ':OUTPATH:';\,
rmf table_testNativeMRJobSimple_input
rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+b = native ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
store b into ':OUTPATH:';\,
'notmq' => 1,
},
@@ -4025,7 +4025,7 @@ rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
b = foreach a generate name;
c = distinct b;
-d = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+d = native ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
e = order d by name;
store e into ':OUTPATH:';\,
'sortArgs' => ['-t', ' '],
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-58
+# TEZ DAG plan: scope-39
#--------------------------------------------------
Tez vertex scope-36 -> Tez vertex scope-37,
Tez vertex scope-38 -> Tez vertex scope-37,
@@ -10,21 +10,21 @@ Tez vertex scope-37
Tez vertex scope-36
# Plan on vertex
-b: Local Rearrange[tuple]{int}(false) - scope-51 -> scope-37
+b: Local Rearrange[tuple]{int}(false) - scope-52 -> scope-37
| |
-| Project[int][0] - scope-53
+| Project[int][0] - scope-54
|
-|---b1: New For Each(false,false)[bag] - scope-39
+|---b1: New For Each(false,false)[bag] - scope-40
| |
- | Project[int][0] - scope-40
+ | Project[int][0] - scope-41
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-41
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-42
| |
- | |---Project[bag][1] - scope-42
+ | |---Project[bag][1] - scope-43
| |
- | |---Project[bag][1] - scope-43
+ | |---Project[bag][1] - scope-44
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-54
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-55
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -56,19 +56,19 @@ Local Rearrange[tuple]{int}(false) - sco
|---c: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-19
Tez vertex scope-37
# Combine plan on edge <scope-36>
-b: Local Rearrange[tuple]{int}(false) - scope-55 -> scope-37
+b: Local Rearrange[tuple]{int}(false) - scope-56 -> scope-37
| |
-| Project[int][0] - scope-57
+| Project[int][0] - scope-58
|
-|---b1: New For Each(false,false)[bag] - scope-44
+|---b1: New For Each(false,false)[bag] - scope-45
| |
- | Project[int][0] - scope-45
+ | Project[int][0] - scope-46
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-46
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-47
| |
- | |---Project[bag][1] - scope-47
+ | |---Project[bag][1] - scope-48
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-50
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-51
# Plan on vertex
d: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-35
|
@@ -84,6 +84,6 @@ d: Store(file:///tmp/output/e:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-16
| |
- | |---Project[bag][1] - scope-48
+ | |---Project[bag][1] - scope-49
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-9
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld Wed Aug 13 19:48:19 2014
@@ -2,26 +2,26 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-39
+# TEZ DAG plan: scope-20
#--------------------------------------------------
Tez vertex scope-18 -> Tez vertex scope-19,
Tez vertex scope-19
Tez vertex scope-18
# Plan on vertex
-b: Local Rearrange[tuple]{int}(false) - scope-32 -> scope-19
+b: Local Rearrange[tuple]{int}(false) - scope-33 -> scope-19
| |
-| Project[int][0] - scope-34
+| Project[int][0] - scope-35
|
-|---c: New For Each(false,false)[bag] - scope-21
+|---c: New For Each(false,false)[bag] - scope-22
| |
- | Project[int][0] - scope-22
+ | Project[int][0] - scope-23
| |
- | POUserFunc(org.apache.pig.builtin.Distinct$Initial)[tuple] - scope-23
+ | POUserFunc(org.apache.pig.builtin.Distinct$Initial)[tuple] - scope-24
| |
- | |---Project[tuple][1] - scope-24
+ | |---Project[tuple][1] - scope-25
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-35
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-36
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -36,19 +36,19 @@ b: Local Rearrange[tuple]{int}(false) -
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-19
# Combine plan on edge <scope-18>
-b: Local Rearrange[tuple]{int}(false) - scope-36 -> scope-19
+b: Local Rearrange[tuple]{int}(false) - scope-37 -> scope-19
| |
-| Project[int][0] - scope-38
+| Project[int][0] - scope-39
|
-|---c: New For Each(false,false)[bag] - scope-25
+|---c: New For Each(false,false)[bag] - scope-26
| |
- | Project[int][0] - scope-26
+ | Project[int][0] - scope-27
| |
- | POUserFunc(org.apache.pig.builtin.Distinct$Intermediate)[tuple] - scope-27
+ | POUserFunc(org.apache.pig.builtin.Distinct$Intermediate)[tuple] - scope-28
| |
- | |---Project[bag][1] - scope-28
+ | |---Project[bag][1] - scope-29
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-31
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-32
# Plan on vertex
c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-17
|
@@ -56,8 +56,8 @@ c: Store(file:///tmp/output:org.apache.p
| |
| POUserFunc(org.apache.pig.builtin.COUNT)[long] - scope-13
| |
- | |---POUserFunc(org.apache.pig.builtin.Distinct$Final)[bag] - scope-20
+ | |---POUserFunc(org.apache.pig.builtin.Distinct$Final)[bag] - scope-21
| |
- | |---Project[bag][1] - scope-29
+ | |---Project[bag][1] - scope-30
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-9
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-29
+# TEZ DAG plan: scope-28
#--------------------------------------------------
Tez vertex scope-26 -> Tez vertex scope-27,
Tez vertex scope-27
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-37
+# TEZ DAG plan: scope-36
#--------------------------------------------------
Tez vertex scope-11 -> Tez vertex scope-30,Tez vertex scope-20,
Tez vertex scope-20 -> Tez vertex scope-30,
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-56
+# TEZ DAG plan: scope-55
#--------------------------------------------------
Tez vertex scope-27 -> Tez vertex scope-46,Tez vertex scope-36,
Tez vertex scope-36 -> Tez vertex scope-46,Tez vertex scope-28,
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld Wed Aug 13 19:48:19 2014
@@ -2,28 +2,28 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-41
+# TEZ DAG plan: scope-22
#--------------------------------------------------
Tez vertex scope-20 -> Tez vertex scope-21,
Tez vertex scope-21
Tez vertex scope-20
# Plan on vertex
-b: Local Rearrange[tuple]{int}(false) - scope-34 -> scope-21
+b: Local Rearrange[tuple]{int}(false) - scope-35 -> scope-21
| |
-| Project[int][0] - scope-36
+| Project[int][0] - scope-37
|
-|---c: New For Each(false,false)[bag] - scope-22
+|---c: New For Each(false,false)[bag] - scope-23
| |
- | Project[int][0] - scope-23
+ | Project[int][0] - scope-24
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-24
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-25
| |
- | |---Project[bag][0] - scope-25
+ | |---Project[bag][0] - scope-26
| |
- | |---Project[bag][1] - scope-26
+ | |---Project[bag][1] - scope-27
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-37
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-38
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -38,19 +38,19 @@ b: Local Rearrange[tuple]{int}(false) -
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-21
# Combine plan on edge <scope-20>
-b: Local Rearrange[tuple]{int}(false) - scope-38 -> scope-21
+b: Local Rearrange[tuple]{int}(false) - scope-39 -> scope-21
| |
-| Project[int][0] - scope-40
+| Project[int][0] - scope-41
|
-|---c: New For Each(false,false)[bag] - scope-27
+|---c: New For Each(false,false)[bag] - scope-28
| |
- | Project[int][0] - scope-28
+ | Project[int][0] - scope-29
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-29
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-30
| |
- | |---Project[bag][1] - scope-30
+ | |---Project[bag][1] - scope-31
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-33
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-34
# Plan on vertex
c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-19
|
@@ -60,6 +60,6 @@ c: Store(file:///tmp/output:org.apache.p
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-16
| |
- | |---Project[bag][1] - scope-31
+ | |---Project[bag][1] - scope-32
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-9
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld Wed Aug 13 19:48:19 2014
@@ -2,13 +2,13 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-33
+# TEZ DAG plan: scope-32
#--------------------------------------------------
Tez vertex scope-24
Tez vertex scope-24
# Plan on vertex
-1-1: Split - scope-32
+1-1: Split - scope-33
| |
| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-13
| |
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-416
+# TEZ DAG plan: scope-393
#--------------------------------------------------
Tez vertex scope-318 -> Tez vertex scope-320,Tez vertex scope-331,Tez vertex scope-342,
Tez vertex scope-320 -> Tez vertex scope-323,Tez vertex scope-336,Tez vertex scope-347,
@@ -11,9 +11,9 @@ Tez vertex scope-376 -> Tez vertex scope
Tez vertex scope-379 -> Tez vertex scope-383,Tez vertex scope-387,
Tez vertex scope-383
Tez vertex scope-342 -> Tez vertex scope-345,Tez vertex scope-385,
-Tez vertex scope-385 -> Tez vertex group scope-413,
-Tez vertex scope-387 -> Tez vertex group scope-413,
-Tez vertex group scope-413
+Tez vertex scope-385 -> Tez vertex group scope-414,
+Tez vertex scope-387 -> Tez vertex group scope-414,
+Tez vertex group scope-414
Tez vertex scope-331 -> Tez vertex scope-334,Tez vertex scope-339,
Tez vertex scope-339 -> Tez vertex scope-341,
Tez vertex scope-341
@@ -113,15 +113,15 @@ POValueOutputTez - scope-344 -> [scope-
|---POValueInputTez - scope-343 <- scope-318
Tez vertex scope-385
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-415
|
|---POValueInputTez - scope-386 <- scope-342
Tez vertex scope-387
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-415
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-416
|
|---POValueInputTez - scope-388 <- scope-379
-Tez vertex group scope-413 <- [scope-385, scope-387] -> null
+Tez vertex group scope-414 <- [scope-385, scope-387] -> null
# No plan on vertex group
Tez vertex scope-331
# Plan on vertex
@@ -138,38 +138,38 @@ POValueOutputTez - scope-333 -> [scope-
|---POValueInputTez - scope-332 <- scope-318
Tez vertex scope-339
# Plan on vertex
-c2: Local Rearrange[tuple]{int}(false) - scope-405 -> scope-341
+c2: Local Rearrange[tuple]{int}(false) - scope-406 -> scope-341
| |
-| Project[int][0] - scope-407
+| Project[int][0] - scope-408
|
-|---c3: New For Each(false,false)[bag] - scope-393
+|---c3: New For Each(false,false)[bag] - scope-394
| |
- | Project[int][0] - scope-394
+ | Project[int][0] - scope-395
| |
- | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-395
+ | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-396
| |
- | |---Project[bag][0] - scope-396
+ | |---Project[bag][0] - scope-397
| |
- | |---Project[bag][1] - scope-397
+ | |---Project[bag][1] - scope-398
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-408
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-409
|
|---POValueInputTez - scope-340 <- scope-331
Tez vertex scope-341
# Combine plan on edge <scope-339>
-c2: Local Rearrange[tuple]{int}(false) - scope-409 -> scope-341
+c2: Local Rearrange[tuple]{int}(false) - scope-410 -> scope-341
| |
-| Project[int][0] - scope-411
+| Project[int][0] - scope-412
|
-|---c3: New For Each(false,false)[bag] - scope-398
+|---c3: New For Each(false,false)[bag] - scope-399
| |
- | Project[int][0] - scope-399
+ | Project[int][0] - scope-400
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-400
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-401
| |
- | |---Project[bag][1] - scope-401
+ | |---Project[bag][1] - scope-402
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-404
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-405
# Plan on vertex
c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-280
|
@@ -179,7 +179,7 @@ c3: Store(file:///tmp/output/c1:org.apac
| |
| POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-277
| |
- | |---Project[bag][1] - scope-402
+ | |---Project[bag][1] - scope-403
|
|---c2: Package(CombinerPackager)[tuple]{int} - scope-270
Tez vertex scope-323
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Wed Aug 13 19:48:19 2014
@@ -2,11 +2,11 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-211
+# TEZ DAG plan: scope-181
#--------------------------------------------------
-Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-208,
-Tez vertex scope-167 -> Tez vertex group scope-208,
-Tez vertex group scope-208
+Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-209,
+Tez vertex scope-167 -> Tez vertex group scope-209,
+Tez vertex group scope-209
Tez vertex scope-113
Tez vertex scope-148 -> Tez vertex scope-158,
Tez vertex scope-158 -> Tez vertex scope-160,
@@ -17,9 +17,9 @@ Tez vertex scope-129
Tez vertex scope-106
# Plan on vertex
-1-1: Split - scope-207
+1-1: Split - scope-208
| |
-| b: Split - scope-204
+| b: Split - scope-205
| | |
| | b1: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-113
| | | |
@@ -29,7 +29,7 @@ Tez vertex scope-106
| | | |
| | | Project[int][0] - scope-50
| | |
-| | 1-2: Split - scope-203
+| | 1-2: Split - scope-204
| | | |
| | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-148
| | | | |
@@ -77,11 +77,11 @@ Tez vertex scope-106
| | |
| | |---Constant(5) - scope-11
| |
-| d1: Split - scope-206
+| d1: Split - scope-207
| | |
| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
| | |
-| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209
+| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
| |
| |---d1: Filter[bag] - scope-73
| | |
@@ -114,20 +114,20 @@ Tez vertex scope-106
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-167
# Plan on vertex
-f1: Split - scope-202
+f1: Split - scope-203
| |
| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
| |
-| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
+| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-211
|
|---f1: Limit - scope-169
|
|---POValueInputTez - scope-168 <- scope-106
-Tez vertex group scope-208 <- [scope-106, scope-167] -> null
+Tez vertex group scope-209 <- [scope-106, scope-167] -> null
# No plan on vertex group
Tez vertex scope-113
# Plan on vertex
-b1: Split - scope-201
+b1: Split - scope-202
| |
| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
| |
@@ -177,27 +177,27 @@ e1: Store(file:///tmp/output/e1:org.apac
|---Package(LitePackager)[tuple]{int} - scope-161
Tez vertex scope-119
# Plan on vertex
-c: Split - scope-205
+c: Split - scope-206
| |
| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
| | |
| | Project[int][0] - scope-48
| |
-| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129
+| c2: Local Rearrange[tuple]{int}(false) - scope-194 -> scope-129
| | |
-| | Project[int][0] - scope-195
+| | Project[int][0] - scope-196
| |
-| |---c3: New For Each(false,false)[bag] - scope-181
+| |---c3: New For Each(false,false)[bag] - scope-182
| | |
-| | Project[int][0] - scope-182
+| | Project[int][0] - scope-183
| | |
-| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183
+| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184
| | |
-| | |---Project[bag][0] - scope-184
+| | |---Project[bag][0] - scope-185
| | |
-| | |---Project[bag][1] - scope-185
+| | |---Project[bag][1] - scope-186
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197
|
|---c: Filter[bag] - scope-34
| |
@@ -221,19 +221,19 @@ c1: Store(file:///tmp/output/c1:org.apac
|---c1: Package(Packager)[tuple]{int} - scope-46
Tez vertex scope-129
# Combine plan on edge <scope-119>
-c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129
+c2: Local Rearrange[tuple]{int}(false) - scope-198 -> scope-129
| |
-| Project[int][0] - scope-199
+| Project[int][0] - scope-200
|
-|---c3: New For Each(false,false)[bag] - scope-186
+|---c3: New For Each(false,false)[bag] - scope-187
| |
- | Project[int][0] - scope-187
+ | Project[int][0] - scope-188
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-188
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189
| |
- | |---Project[bag][1] - scope-189
+ | |---Project[bag][1] - scope-190
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-192
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-193
# Plan on vertex
c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68
|
@@ -243,6 +243,6 @@ c3: Store(file:///tmp/output/c1:org.apac
| |
| POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65
| |
- | |---Project[bag][1] - scope-190
+ | |---Project[bag][1] - scope-191
|
|---c2: Package(CombinerPackager)[tuple]{int} - scope-58
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-174
+# TEZ DAG plan: scope-134
#--------------------------------------------------
Tez vertex scope-126 -> Tez vertex scope-128,Tez vertex scope-131,
Tez vertex scope-131 -> Tez vertex scope-133,
@@ -27,38 +27,38 @@ POValueOutputTez - scope-127 -> [scope-
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-88
Tez vertex scope-131
# Plan on vertex
-c: Local Rearrange[tuple]{tuple}(false) - scope-165 -> scope-133
+c: Local Rearrange[tuple]{tuple}(false) - scope-166 -> scope-133
| |
-| Project[tuple][0] - scope-168
+| Project[tuple][0] - scope-169
|
-|---c: New For Each(false,false)[bag] - scope-153
+|---c: New For Each(false,false)[bag] - scope-154
| |
- | Project[tuple][0] - scope-154
+ | Project[tuple][0] - scope-155
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-155
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-156
| |
- | |---Project[bag][1] - scope-156
+ | |---Project[bag][1] - scope-157
| |
- | |---Project[bag][1] - scope-157
+ | |---Project[bag][1] - scope-158
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-169
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-170
|
|---POValueInputTez - scope-132 <- scope-126
Tez vertex scope-133
# Combine plan on edge <scope-131>
-c: Local Rearrange[tuple]{tuple}(false) - scope-170 -> scope-133
+c: Local Rearrange[tuple]{tuple}(false) - scope-171 -> scope-133
| |
-| Project[tuple][0] - scope-173
+| Project[tuple][0] - scope-174
|
-|---c: New For Each(false,false)[bag] - scope-158
+|---c: New For Each(false,false)[bag] - scope-159
| |
- | Project[tuple][0] - scope-159
+ | Project[tuple][0] - scope-160
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-160
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-161
| |
- | |---Project[bag][1] - scope-161
+ | |---Project[bag][1] - scope-162
|
- |---c: Package(CombinerPackager)[tuple]{tuple} - scope-164
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-165
# Plan on vertex
c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-125
|
@@ -68,43 +68,43 @@ c: Store(file:///tmp/output/c:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-122
| |
- | |---Project[bag][1] - scope-162
+ | |---Project[bag][1] - scope-163
|
|---c: Package(CombinerPackager)[tuple]{tuple} - scope-114
Tez vertex scope-128
# Plan on vertex
-b: Local Rearrange[tuple]{int}(false) - scope-146 -> scope-130
+b: Local Rearrange[tuple]{int}(false) - scope-147 -> scope-130
| |
-| Project[int][0] - scope-148
+| Project[int][0] - scope-149
|
-|---b: New For Each(false,false)[bag] - scope-134
+|---b: New For Each(false,false)[bag] - scope-135
| |
- | Project[int][0] - scope-135
+ | Project[int][0] - scope-136
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-136
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-137
| |
- | |---Project[bag][0] - scope-137
+ | |---Project[bag][0] - scope-138
| |
- | |---Project[bag][1] - scope-138
+ | |---Project[bag][1] - scope-139
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-149
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-150
|
|---POValueInputTez - scope-129 <- scope-126
Tez vertex scope-130
# Combine plan on edge <scope-128>
-b: Local Rearrange[tuple]{int}(false) - scope-150 -> scope-130
+b: Local Rearrange[tuple]{int}(false) - scope-151 -> scope-130
| |
-| Project[int][0] - scope-152
+| Project[int][0] - scope-153
|
-|---b: New For Each(false,false)[bag] - scope-139
+|---b: New For Each(false,false)[bag] - scope-140
| |
- | Project[int][0] - scope-140
+ | Project[int][0] - scope-141
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-141
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-142
| |
- | |---Project[bag][1] - scope-142
+ | |---Project[bag][1] - scope-143
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-145
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-146
# Plan on vertex
b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-110
|
@@ -114,6 +114,6 @@ b: Store(file:///tmp/output/b:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-107
| |
- | |---Project[bag][1] - scope-143
+ | |---Project[bag][1] - scope-144
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-100
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-87
+# TEZ DAG plan: scope-46
#--------------------------------------------------
Tez vertex scope-38 -> Tez vertex scope-42,Tez vertex scope-45,
Tez vertex scope-45
@@ -10,39 +10,39 @@ Tez vertex scope-42
Tez vertex scope-38
# Plan on vertex
-a: Split - scope-86
+a: Split - scope-87
| |
-| b: Local Rearrange[tuple]{int}(false) - scope-58 -> scope-42
+| b: Local Rearrange[tuple]{int}(false) - scope-59 -> scope-42
| | |
-| | Project[int][0] - scope-60
+| | Project[int][0] - scope-61
| |
-| |---b: New For Each(false,false)[bag] - scope-46
+| |---b: New For Each(false,false)[bag] - scope-47
| | |
-| | Project[int][0] - scope-47
+| | Project[int][0] - scope-48
| | |
-| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-48
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-49
| | |
-| | |---Project[bag][0] - scope-49
+| | |---Project[bag][0] - scope-50
| | |
-| | |---Project[bag][1] - scope-50
+| | |---Project[bag][1] - scope-51
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-61
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-62
| |
-| c: Local Rearrange[tuple]{tuple}(false) - scope-77 -> scope-45
+| c: Local Rearrange[tuple]{tuple}(false) - scope-78 -> scope-45
| | |
-| | Project[tuple][0] - scope-80
+| | Project[tuple][0] - scope-81
| |
-| |---c: New For Each(false,false)[bag] - scope-65
+| |---c: New For Each(false,false)[bag] - scope-66
| | |
-| | Project[tuple][0] - scope-66
+| | Project[tuple][0] - scope-67
| | |
-| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-67
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-68
| | |
-| | |---Project[bag][1] - scope-68
+| | |---Project[bag][1] - scope-69
| | |
-| | |---Project[bag][1] - scope-69
+| | |---Project[bag][1] - scope-70
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-81
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-82
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -57,19 +57,19 @@ a: Split - scope-86
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-45
# Combine plan on edge <scope-38>
-c: Local Rearrange[tuple]{tuple}(false) - scope-82 -> scope-45
+c: Local Rearrange[tuple]{tuple}(false) - scope-83 -> scope-45
| |
-| Project[tuple][0] - scope-85
+| Project[tuple][0] - scope-86
|
-|---c: New For Each(false,false)[bag] - scope-70
+|---c: New For Each(false,false)[bag] - scope-71
| |
- | Project[tuple][0] - scope-71
+ | Project[tuple][0] - scope-72
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-72
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-73
| |
- | |---Project[bag][1] - scope-73
+ | |---Project[bag][1] - scope-74
|
- |---c: Package(CombinerPackager)[tuple]{tuple} - scope-76
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-77
# Plan on vertex
c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-37
|
@@ -79,24 +79,24 @@ c: Store(file:///tmp/output/c:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-34
| |
- | |---Project[bag][1] - scope-74
+ | |---Project[bag][1] - scope-75
|
|---c: Package(CombinerPackager)[tuple]{tuple} - scope-26
Tez vertex scope-42
# Combine plan on edge <scope-38>
-b: Local Rearrange[tuple]{int}(false) - scope-62 -> scope-42
+b: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-42
| |
-| Project[int][0] - scope-64
+| Project[int][0] - scope-65
|
-|---b: New For Each(false,false)[bag] - scope-51
+|---b: New For Each(false,false)[bag] - scope-52
| |
- | Project[int][0] - scope-52
+ | Project[int][0] - scope-53
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-53
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-54
| |
- | |---Project[bag][1] - scope-54
+ | |---Project[bag][1] - scope-55
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-57
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-58
# Plan on vertex
b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-22
|
@@ -106,6 +106,6 @@ b: Store(file:///tmp/output/b:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-19
| |
- | |---Project[bag][1] - scope-55
+ | |---Project[bag][1] - scope-56
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-12
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-64
+# TEZ DAG plan: scope-63
#--------------------------------------------------
Tez vertex scope-53 -> Tez vertex scope-55,
Tez vertex scope-54 -> Tez vertex scope-55,
@@ -44,7 +44,7 @@ c: Local Rearrange[tuple]{int}(false) -
|---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
Tez vertex scope-55
# Plan on vertex
-c: Split - scope-63
+c: Split - scope-64
| |
| c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-30
| |
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-59
+# TEZ DAG plan: scope-57
#--------------------------------------------------
Tez vertex scope-43 -> Tez vertex scope-44,
Tez vertex scope-44
@@ -26,7 +26,7 @@ b: Local Rearrange[tuple]{int}(false) -
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-44
# Plan on vertex
-b: Split - scope-58
+b: Split - scope-59
| |
| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-15
| |
@@ -42,7 +42,7 @@ b: Split - scope-58
| | |
| | |---Project[bag][1] - scope-21
| |
-| d: Split - scope-57
+| d: Split - scope-58
| | |
| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-34
| | |
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld Wed Aug 13 19:48:19 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-53
+# TEZ DAG plan: scope-52
#--------------------------------------------------
Tez vertex scope-23 -> Tez vertex scope-24,
Tez vertex scope-24 -> Tez vertex scope-43,Tez vertex scope-33,
Modified: pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld?rev=1617821&r1=1617820&r2=1617821&view=diff
==============================================================================
--- pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld (original)
+++ pig/branches/maven/pig/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld Wed Aug 13 19:48:19 2014
@@ -2,15 +2,15 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-27
+# TEZ DAG plan: scope-24
#--------------------------------------------------
-Tez vertex scope-18 -> Tez vertex group scope-24,
-Tez vertex scope-19 -> Tez vertex group scope-24,
-Tez vertex group scope-24
+Tez vertex scope-18 -> Tez vertex group scope-25,
+Tez vertex scope-19 -> Tez vertex group scope-25,
+Tez vertex group scope-25
Tez vertex scope-18
# Plan on vertex
-c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -25,7 +25,7 @@ c: Store(file:///tmp/output:org.apache.p
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-19
# Plan on vertex
-c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-27
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -38,5 +38,5 @@ c: Store(file:///tmp/output:org.apache.p
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-24 <- [scope-18, scope-19] -> null
+Tez vertex group scope-25 <- [scope-18, scope-19] -> null
# No plan on vertex group