You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/12/03 19:07:51 UTC
svn commit: r1547505 [2/2] - in /pig/branches/tez: src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/backend/hadoop/executionengine/...
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Tue Dec 3 18:07:50 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.Input
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -51,6 +52,7 @@ import org.apache.pig.backend.hadoop.HDa
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PhyPlanSetter;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBigDecimalRawComparator;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBigIntegerRawComparator;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigBooleanRawComparator;
@@ -87,15 +89,16 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.UDFContext;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.mapreduce.combine.MRCombiner;
-import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
+import org.apache.tez.mapreduce.common.MRInputSplitDistributor;
+import org.apache.tez.mapreduce.hadoop.InputSplitInfo;
import org.apache.tez.mapreduce.hadoop.MRHelpers;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.input.MRInput;
@@ -108,7 +111,7 @@ import org.apache.tez.mapreduce.partitio
public class TezDagBuilder extends TezOpPlanVisitor {
private static final Log log = LogFactory.getLog(TezJobControlCompiler.class);
- private DAG dag;
+ private TezDAG dag;
private Map<String, LocalResource> localResources;
private PigContext pc;
private Configuration globalConf;
@@ -116,7 +119,7 @@ public class TezDagBuilder extends TezOp
private String scope;
private NodeIdGenerator nig;
- public TezDagBuilder(PigContext pc, TezOperPlan plan, DAG dag,
+ public TezDagBuilder(PigContext pc, TezOperPlan plan, TezDAG dag,
Map<String, LocalResource> localResources) {
super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan));
this.pc = pc;
@@ -134,7 +137,7 @@ public class TezDagBuilder extends TezOp
try {
to = newVertex(tezOp);
dag.addVertex(to);
- } catch (IOException e) {
+ } catch (Exception e) {
throw new VisitorException("Cannot create vertex for "
+ tezOp.name(), e);
}
@@ -146,7 +149,7 @@ public class TezDagBuilder extends TezOp
for (TezOperator predecessor : predecessors) {
// Since this is a dependency order walker, predecessor vertices
// must have already been created.
- Vertex from = dag.getVertex(predecessor.name());
+ Vertex from = dag.getVertex(predecessor.getOperatorKey().toString());
EdgeProperty prop = null;
try {
prop = newEdge(predecessor, tezOp);
@@ -176,19 +179,34 @@ public class TezDagBuilder extends TezOp
InputDescriptor in = new InputDescriptor(edge.inputClassName);
OutputDescriptor out = new OutputDescriptor(edge.outputClassName);
+ Configuration conf = new Configuration(false);
if (!combinePlan.isEmpty()) {
- Configuration conf = new Configuration();
- addCombiner(combinePlan, conf);
- in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
- out.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+ addCombiner(combinePlan, to, conf);
}
+ List<POLocalRearrangeTez> lrs = PlanHelper.getPhysicalOperators(from.plan,
+ POLocalRearrangeTez.class);
+
+ for (POLocalRearrangeTez lr : lrs) {
+ if (lr.getOutputKey().equals(to.getOperatorKey().toString())) {
+ byte keyType = lr.getKeyType();
+ setIntermediateInputKeyValue(keyType, conf);
+ setIntermediateOutputKeyValue(keyType, conf);
+ conf.set("pig.reduce.key.type", Byte.toString(keyType));
+ break;
+ }
+ }
+
+ in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+ out.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
+
return new EdgeProperty(edge.dataMovementType, edge.dataSourceType,
edge.schedulingType, out, in);
}
- private void addCombiner(PhysicalPlan combinePlan, Configuration conf) throws IOException {
- POPackage combPack = (POPackage)combinePlan.getRoots().get(0);
+ private void addCombiner(PhysicalPlan combinePlan, TezOperator pkgTezOp,
+ Configuration conf) throws IOException {
+ POPackage combPack = (POPackage) combinePlan.getRoots().get(0);
setIntermediateInputKeyValue(combPack.getPkgr().getKeyType(), conf);
POLocalRearrange combRearrange = (POLocalRearrange) combinePlan
@@ -196,7 +214,7 @@ public class TezDagBuilder extends TezOp
setIntermediateOutputKeyValue(combRearrange.getKeyType(), conf);
LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(
- combinePlan, combPack);
+ combinePlan, pkgTezOp, combPack);
lrDiscoverer.visit();
combinePlan.remove(combPack);
@@ -212,21 +230,21 @@ public class TezDagBuilder extends TezOp
.serialize(new byte[] { combRearrange.getKeyType() }));
}
- private Vertex newVertex(TezOperator tezOp) throws IOException {
+ private Vertex newVertex(TezOperator tezOp) throws IOException,
+ ClassNotFoundException, InterruptedException {
ProcessorDescriptor procDesc = new ProcessorDescriptor(
tezOp.getProcessorName());
- // Pass physical plans to vertex as user payload.
- Configuration payloadConf = new Configuration();
- // We won't actually use this job, but we need it to talk with the Load
- // Store funcs
+ // We won't actually use this job, but we need it to talk with the Load Store funcs
@SuppressWarnings("deprecation")
- Job job = new Job(payloadConf);
+ Job job = new Job();
- ArrayList<POStore> storeLocations = new ArrayList<POStore>();
- Path tmpLocation = null;
+ // Pass physical plans to vertex as user payload.
+ Configuration payloadConf = job.getConfiguration();
List<POLoad> loads = processLoads(tezOp, payloadConf, job);
+ LinkedList<POStore> stores = processStores(tezOp, payloadConf, job);
+
payloadConf.set("pig.pigContext", ObjectSerializer.serialize(pc));
payloadConf.set("udf.import.list",
@@ -235,87 +253,6 @@ public class TezDagBuilder extends TezOp
payloadConf.setClass("mapreduce.inputformat.class",
PigInputFormat.class, InputFormat.class);
- // We need to remove all the POStores from the exec plan and later add
- // them as outputs of the vertex
- LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
- tezOp.plan, POStore.class);
-
- for (POStore st : stores) {
- storeLocations.add(st);
- StoreFuncInterface sFunc = st.getStoreFunc();
- sFunc.setStoreLocation(st.getSFile().getFileName(), job);
- }
-
- if (stores.size() == 1) {
- POStore st = stores.get(0);
- if (!pc.inIllustrator)
- tezOp.plan.remove(st);
-
- // set out filespecs
- String outputPathString = st.getSFile().getFileName();
- if (!outputPathString.contains("://")
- || outputPathString.startsWith("hdfs://")) {
- payloadConf.set("pig.streaming.log.dir", new Path(
- outputPathString, JobControlCompiler.LOG_DIR)
- .toString());
- } else {
- String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
- .toString();
- tmpLocation = new Path(tmpLocationStr);
- payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
- JobControlCompiler.LOG_DIR).toString());
- }
- payloadConf.set("pig.streaming.task.output.dir", outputPathString);
- } else if (stores.size() > 0) { // multi store case
- log.info("Setting up multi store job");
- String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
- .toString();
- tmpLocation = new Path(tmpLocationStr);
-
- boolean disableCounter = payloadConf.getBoolean(
- "pig.disable.counter", false);
- if (disableCounter) {
- log.info("Disable Pig custom output counters");
- }
- int idx = 0;
- for (POStore sto : storeLocations) {
- sto.setDisableCounter(disableCounter);
- sto.setMultiStore(true);
- sto.setIndex(idx++);
- }
-
- payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
- JobControlCompiler.LOG_DIR).toString());
- payloadConf.set("pig.streaming.task.output.dir",
- tmpLocation.toString());
- }
-
- if (!pc.inIllustrator) {
- // Unset inputs for POStore, otherwise, exec plan will be
- // unnecessarily deserialized
- for (POStore st : stores) {
- st.setInputs(null);
- st.setParentPlan(null);
- }
- // We put them in the reduce because PigOutputCommitter checks the
- // ID of the task to see if it's a map, and if not, calls the reduce
- // committers.
- payloadConf.set(JobControlCompiler.PIG_MAP_STORES,
- ObjectSerializer.serialize(new ArrayList<POStore>()));
- payloadConf.set(JobControlCompiler.PIG_REDUCE_STORES,
- ObjectSerializer.serialize(stores));
- }
-
- // For all shuffle outputs, configure the classes
- List<PhysicalOperator> leaves = tezOp.plan.getLeaves();
- // TODO: For multiple POLocalRearrange leaves, we need to loop over
- // leaves and set up per shuffle. i.e. SPLIT + multiple GROUP BY with
- // different keys.
- if (leaves.size() == 1 && leaves.get(0) instanceof POLocalRearrange) {
- byte keyType = ((POLocalRearrange)leaves.get(0)).getKeyType();
- setIntermediateOutputKeyValue(keyType, payloadConf);
- payloadConf.set("pig.reduce.key.type", Byte.toString(keyType));
- }
// Configure the classes for incoming shuffles to this TezOp
List<PhysicalOperator> roots = tezOp.plan.getRoots();
@@ -382,6 +319,10 @@ public class TezDagBuilder extends TezOp
payloadConf.setClass("mapreduce.outputformat.class",
PigOutputFormat.class, OutputFormat.class);
+ // set parent plan in all operators.
+ // currently the parent plan is really used only when POStream, POSplit are present in the plan
+ new PhyPlanSetter(tezOp.plan).visit();
+
// Serialize the execution plan
payloadConf.set(PigProcessor.PLAN,
ObjectSerializer.serialize(tezOp.plan));
@@ -392,8 +333,17 @@ public class TezDagBuilder extends TezOp
procDesc.setUserPayload(userPayload);
// Can only set parallelism here if the parallelism isn't derived from
// splits
- int parallelism = (loads.size() == 0) ? tezOp.requestedParallelism : -1;
- Vertex vertex = new Vertex(tezOp.name(), procDesc, parallelism,
+ int parallelism = tezOp.requestedParallelism;
+ InputSplitInfo inputSplitInfo = null;
+ if (loads != null && loads.size() > 0) {
+ // Not using MRInputAMSplitGenerator because delegation tokens are
+ // fetched in FileInputFormat
+ inputSplitInfo = MRHelpers.generateInputSplitsToMem(payloadConf);
+ // TODO: Can be set to -1 if TEZ-601 gets fixed and getting input
+ // splits can be moved to if(loads) block below
+ parallelism = inputSplitInfo.getNumTasks();
+ }
+ Vertex vertex = new Vertex(tezOp.getOperatorKey().toString(), procDesc, parallelism,
Resource.newInstance(tezOp.requestedMemory, tezOp.requestedCpu));
Map<String, String> env = new HashMap<String, String>();
@@ -413,16 +363,55 @@ public class TezDagBuilder extends TezOp
// TODO: These should get the globalConf, or a merged version that
// keeps settings like pig.maxCombinedSplitSize
+ vertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
vertex.addInput(ld.getOperatorKey().toString(),
new InputDescriptor(MRInput.class.getName())
.setUserPayload(MRHelpers.createMRInputPayload(
- userPayload, null)),
- MRInputAMSplitGenerator.class);
- }
- if (!stores.isEmpty()) {
- vertex.addOutput("PigOutput",
- new OutputDescriptor(MROutput.class.getName()));
- }
+ userPayload,
+ inputSplitInfo.getSplitsProto())),
+ MRInputSplitDistributor.class);
+ }
+
+ for (POStore store : stores) {
+
+ ArrayList<POStore> emptyList = new ArrayList<POStore>();
+ ArrayList<POStore> singleStore = new ArrayList<POStore>();
+ singleStore.add(store);
+
+ Configuration outputPayLoad = new Configuration(payloadConf);
+ outputPayLoad.set(JobControlCompiler.PIG_MAP_STORES,
+ ObjectSerializer.serialize(emptyList));
+ outputPayLoad.set(JobControlCompiler.PIG_REDUCE_STORES,
+ ObjectSerializer.serialize(singleStore));
+
+ vertex.addOutput(store.getOperatorKey().toString(),
+ new OutputDescriptor(MROutput.class.getName())
+ .setUserPayload(TezUtils.createUserPayloadFromConf(outputPayLoad)));
+ }
+
+ if (stores.size() > 0) {
+ new PigOutputFormat().checkOutputSpecs(job);
+ }
+
+ // LoadFunc and StoreFunc add delegation tokens to Job Credentials in
+ // setLocation and setStoreLocation respectively. For eg: HBaseStorage
+ // InputFormat add delegation token in getSplits and OutputFormat in
+ // checkOutputSpecs. For eg: FileInputFormat and FileOutputFormat
+ dag.getCredentials().addAll(job.getCredentials());
+
+ // Add credentials from binary token file and get tokens for namenodes
+ // specified in mapreduce.job.hdfs-servers
+ SecurityHelper.populateTokenCache(job.getConfiguration(), dag.getCredentials());
+
+ // Unlike MR which gets staging dir from RM, tez AM is picking it up
+ // from configuration
+ Path submitJobDir = new Path(payloadConf.get(
+ TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
+ // Get delegation token for the job submission dir
+ TokenCache.obtainTokensForNamenodes(dag.getCredentials(),
+ new Path[] { submitJobDir }, payloadConf);
+
return vertex;
}
@@ -495,6 +484,78 @@ public class TezDagBuilder extends TezOp
return lds;
}
+ private LinkedList<POStore> processStores(TezOperator tezOp,
+ Configuration payloadConf, Job job) throws VisitorException,
+ IOException {
+ LinkedList<POStore> stores = PlanHelper.getPhysicalOperators(
+ tezOp.plan, POStore.class);
+ if (stores.size() > 0) {
+
+ ArrayList<POStore> storeLocations = new ArrayList<POStore>();
+ for (POStore st : stores) {
+ storeLocations.add(st);
+ StoreFuncInterface sFunc = st.getStoreFunc();
+ sFunc.setStoreLocation(st.getSFile().getFileName(), job);
+ }
+
+ Path tmpLocation = null;
+ if (stores.size() == 1) {
+ POStore st = stores.get(0);
+
+ // set out filespecs
+ String outputPathString = st.getSFile().getFileName();
+ if (!outputPathString.contains("://")
+ || outputPathString.startsWith("hdfs://")) {
+ payloadConf.set("pig.streaming.log.dir", new Path(
+ outputPathString, JobControlCompiler.LOG_DIR)
+ .toString());
+ } else {
+ String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
+ .toString();
+ tmpLocation = new Path(tmpLocationStr);
+ payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
+ JobControlCompiler.LOG_DIR).toString());
+ }
+ payloadConf.set("pig.streaming.task.output.dir", outputPathString);
+ } else { // multi store case
+ log.info("Setting up multi store job");
+ String tmpLocationStr = FileLocalizer.getTemporaryPath(pc)
+ .toString();
+ tmpLocation = new Path(tmpLocationStr);
+
+ boolean disableCounter = payloadConf.getBoolean(
+ "pig.disable.counter", false);
+ if (disableCounter) {
+ log.info("Disable Pig custom output counters");
+ }
+ int idx = 0;
+ for (POStore sto : storeLocations) {
+ sto.setDisableCounter(disableCounter);
+ sto.setMultiStore(true);
+ sto.setIndex(idx++);
+ }
+
+ payloadConf.set("pig.streaming.log.dir", new Path(tmpLocation,
+ JobControlCompiler.LOG_DIR).toString());
+ payloadConf.set("pig.streaming.task.output.dir",
+ tmpLocation.toString());
+ }
+
+ if (!pc.inIllustrator) {
+
+ // We put them in the reduce because PigOutputCommitter checks the
+ // ID of the task to see if it's a map, and if not, calls the reduce
+ // committers.
+ payloadConf.set(JobControlCompiler.PIG_MAP_STORES,
+ ObjectSerializer.serialize(new ArrayList<POStore>()));
+ payloadConf.set(JobControlCompiler.PIG_REDUCE_STORES,
+ ObjectSerializer.serialize(stores));
+ }
+
+ }
+ return stores;
+ }
+
@SuppressWarnings("rawtypes")
private void setIntermediateInputKeyValue(byte keyType, Configuration conf)
throws JobCreationException, ExecException {
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java Tue Dec 3 18:07:50 2013
@@ -41,13 +41,13 @@ public class TezJob extends ControlledJo
private static final Log log = LogFactory.getLog(TezJob.class);
private DAGStatus dagStatus;
private Configuration conf;
- private DAG dag;
+ private TezDAG dag;
private DAGClient dagClient;
private Map<String, LocalResource> requestAMResources;
private TezSession tezSession;
private boolean reuseSession;
- public TezJob(TezConfiguration conf, DAG dag, Map<String, LocalResource> requestAMResources)
+ public TezJob(TezConfiguration conf, TezDAG dag, Map<String, LocalResource> requestAMResources)
throws IOException {
super(conf);
this.conf = conf;
@@ -67,12 +67,14 @@ public class TezJob extends ControlledJo
@Override
public void submit() {
try {
- tezSession = TezSessionManager.getSession(conf, requestAMResources);
+ tezSession = TezSessionManager.getSession(conf, requestAMResources, dag.getCredentials());
log.info("Submitting DAG - Application id: " + tezSession.getApplicationId());
dagClient = tezSession.submitDAG(dag);
} catch (Exception e) {
- if (tezSession!=null) {
- log.info("Cannot submit DAG - Application id: " + tezSession.getApplicationId(), e);
+ if (tezSession != null) {
+ log.error("Cannot submit DAG - Application id: " + tezSession.getApplicationId(), e);
+ } else {
+ log.error("Cannot submit DAG", e);
}
setJobState(ControlledJob.State.FAILED);
return;
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java Tue Dec 3 18:07:50 2013
@@ -23,13 +23,12 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigException;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
import org.apache.pig.impl.PigContext;
-import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.TezConfiguration;
/**
@@ -48,10 +47,10 @@ public class TezJobControlCompiler {
this.tezConf = new TezConfiguration(conf);
}
- public DAG buildDAG(TezOperPlan tezPlan, Map<String, LocalResource> localResources)
+ public TezDAG buildDAG(TezOperPlan tezPlan, Map<String, LocalResource> localResources)
throws IOException, YarnException {
String jobName = pigContext.getProperties().getProperty(PigContext.JOB_NAME, "pig");
- DAG tezDag = new DAG(jobName);
+ TezDAG tezDag = new TezDAG(jobName);
TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlan, tezDag, localResources);
dagBuilder.visit();
return tezDag;
@@ -78,7 +77,7 @@ public class TezJobControlCompiler {
TezJobControl jobCtrl = new TezJobControl(grpName, timeToSleep);
try {
- // A single Tez job always pack only 1 Tez plan. We will track
+ // A single Tez job always pack only 1 Tez plan. We will track
// Tez job asynchronously to exploit parallel execution opportunities.
TezJob job = getJob(tezPlan, planContainer);
jobCtrl.addJob(job);
@@ -99,7 +98,7 @@ public class TezJobControlCompiler {
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
localResources.putAll(planContainer.getLocalResources());
localResources.putAll(tezPlan.getLocalExtraResources());
- DAG tezDag = buildDAG(tezPlan, localResources);
+ TezDAG tezDag = buildDAG(tezPlan, localResources);
return new TezJob(tezConf, tezDag, planContainer.getLocalResources());
} catch (Exception e) {
int errCode = 2017;
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOperator.java Tue Dec 3 18:07:50 2013
@@ -62,26 +62,30 @@ public class TezOperator extends Operato
// Name of the Custom Partitioner used
String customPartitioner = null;
+ // Presence indicates that this TezOper is sub-plan of a POSplit.
+ // Only POStore or POLocalRearrange leaf can be a sub-plan of POSplit
+ private OperatorKey splitOperatorKey = null;
+
// Indicates that the plan creation is complete
boolean closed = false;
-
+
// Indicate whether we need to split the DAG below the operator
// The result is two or more DAG connected DAG inside the same plan container
boolean segmentBelow = false;
-
+
// Indicates if this is a limit after a sort
boolean limitAfterSort = false;
-
+
//Indicates if this job is an order by job
boolean globalSort = false;
-
+
//The quantiles file name if globalSort is true
String quantFile;
-
+
//The sort order of the columns;
//asc is true and desc is false
boolean[] sortOrder;
-
+
// Last POLimit value in this map reduce operator, needed by LimitAdjuster
// to add additional map reduce operator with 1 reducer after this
long limit = -1;
@@ -119,14 +123,28 @@ public class TezOperator extends Operato
v.visitTezOp(this);
}
+ @Override
public boolean supportsMultipleInputs() {
return true;
}
+ @Override
public boolean supportsMultipleOutputs() {
return true;
}
+ public OperatorKey getSplitOperatorKey() {
+ return splitOperatorKey;
+ }
+
+ public void setSplitOperatorKey(OperatorKey splitOperatorKey) {
+ this.splitOperatorKey = splitOperatorKey;
+ }
+
+ public boolean isSplitSubPlan() {
+ return splitOperatorKey != null;
+ }
+
public boolean isClosed() {
return closed;
}
@@ -216,7 +234,7 @@ public class TezOperator extends Operato
public boolean needSegmentBelow() {
return segmentBelow;
}
-
+
public String getQuantFile() {
return quantFile;
}
@@ -232,19 +250,19 @@ public class TezOperator extends Operato
this.sortOrder[i] = sortOrder[i];
}
}
-
+
public boolean[] getSortOrder() {
return sortOrder;
}
-
+
public void setGlobalSort(boolean globalSort) {
this.globalSort = globalSort;
}
-
+
public boolean isGlobalSort() {
return globalSort;
}
-
+
public boolean isLimitAfterSort() {
return limitAfterSort;
}
Added: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java (added)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezOutput.java Tue Dec 3 18:07:50 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.executionengine.tez;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.tez.runtime.api.LogicalOutput;
+
+/**
+ * This interface is implemented by PhysicalOperators that can have Tez outputs
+ * attached directly to the operator.
+ */
+
+public interface TezOutput {
+ public void attachOutputs(Map<String, LogicalOutput> outputs,
+ Configuration conf) throws ExecException;
+}
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java Tue Dec 3 18:07:50 2013
@@ -60,13 +60,13 @@ public class TezPOPackageAnnotator exten
}
}
- private void handlePackage(TezOperator tezOp, POPackage pkg) throws VisitorException {
+ private void handlePackage(TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
// the LocalRearrange(s) must be in the plan of a predecessor tez op
int lrFound = 0;
- List<TezOperator> preds = this.mPlan.getPredecessors(tezOp);
+ List<TezOperator> preds = this.mPlan.getPredecessors(pkgTezOp);
for (Iterator<TezOperator> it = preds.iterator(); it.hasNext();) {
- TezOperator tezOper = it.next();
- lrFound += patchPackage(tezOper.plan, pkg);
+ TezOperator predTezOp = it.next();
+ lrFound += patchPackage(predTezOp, pkgTezOp, pkg);
if(lrFound == pkg.getNumInps()) {
break;
}
@@ -79,8 +79,8 @@ public class TezPOPackageAnnotator exten
}
}
- private int patchPackage(PhysicalPlan plan, POPackage pkg) throws VisitorException {
- LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(plan, pkg);
+ private int patchPackage(TezOperator predTezOp, TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
+ LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(predTezOp.plan, pkgTezOp, pkg);
lrDiscoverer.visit();
// let our caller know if we managed to patch
// the package
@@ -124,15 +124,21 @@ public class TezPOPackageAnnotator exten
static class LoRearrangeDiscoverer extends PhyPlanVisitor {
private int loRearrangeFound = 0;
+ private TezOperator pkgTezOp;
private POPackage pkg;
- public LoRearrangeDiscoverer(PhysicalPlan plan, POPackage pkg) {
+ public LoRearrangeDiscoverer(PhysicalPlan plan, TezOperator pkgTezOp, POPackage pkg) {
super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
+ this.pkgTezOp = pkgTezOp;
this.pkg = pkg;
}
@Override
public void visitLocalRearrange(POLocalRearrange lrearrange) throws VisitorException {
+ POLocalRearrangeTez lr = (POLocalRearrangeTez) lrearrange;
+ if (!lr.getOutputKey().equals(pkgTezOp.getOperatorKey().toString())) {
+ return;
+ }
loRearrangeFound++;
Map<Integer,Pair<Boolean, Map<Integer, Integer>>> keyInfo;
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java Tue Dec 3 18:07:50 2013
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.impl.PigContext;
@@ -75,7 +76,7 @@ public class TezSessionManager {
}
}
- private static SessionInfo createSession(Configuration conf, Map<String, LocalResource> requestedAMResources) throws TezException, IOException {
+ private static SessionInfo createSession(Configuration conf, Map<String, LocalResource> requestedAMResources, Credentials creds) throws TezException, IOException {
TezConfiguration tezConf = new TezConfiguration(conf);
TezClient tezClient = new TezClient(tezConf);
ApplicationId appId = tezClient.createApplication();
@@ -84,7 +85,7 @@ public class TezSessionManager {
resources.putAll(requestedAMResources);
String jobName = conf.get(PigContext.JOB_NAME, "pig");
- AMConfiguration amConfig = new AMConfiguration(null, resources, tezConf, null);
+ AMConfiguration amConfig = new AMConfiguration(null, resources, tezConf, creds);
TezSessionConfiguration sessionConfig = new TezSessionConfiguration(amConfig, tezConf);
TezSession tezSession = new TezSession(jobName, appId, sessionConfig);
tezSession.start();
@@ -101,7 +102,7 @@ public class TezSessionManager {
return true;
}
- static TezSession getSession(Configuration conf, Map<String, LocalResource> requestedAMResources) throws TezException, IOException {
+ static TezSession getSession(Configuration conf, Map<String, LocalResource> requestedAMResources, Credentials creds) throws TezException, IOException {
synchronized (sessionPool) {
List<SessionInfo> sessionsToRemove = new ArrayList<SessionInfo>();
for (SessionInfo sessionInfo : sessionPool) {
@@ -120,7 +121,7 @@ public class TezSessionManager {
}
// We cannot find available AM, create new one
- SessionInfo sessionInfo = createSession(conf, requestedAMResources);
+ SessionInfo sessionInfo = createSession(conf, requestedAMResources, creds);
sessionInfo.inUse = true;
sessionPool.add(sessionInfo);
return sessionInfo.session;
Modified: pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/pigstats/tez/TezStats.java Tue Dec 3 18:07:50 2013
@@ -74,7 +74,7 @@ public class TezStats extends PigStats {
}
}
}
- tezOpVertexMap.put(tezOp.name(), currStats);
+ tezOpVertexMap.put(tezOp.getOperatorKey().toString(), currStats);
}
}
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld Tue Dec 3 18:07:50 2013
@@ -0,0 +1,51 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-148
+#--------------------------------------------------
+Tez vertex scope-142
+# Plan on vertex
+6-1: Split - scope-126
+| |
+| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-131
+| |
+| |---b: Filter[bag] - scope-127
+| | |
+| | Less Than or Equal[boolean] - scope-130
+| | |
+| | |---Project[int][0] - scope-128
+| | |
+| | |---Constant(5) - scope-129
+| |
+| c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-136
+| |
+| |---c: Filter[bag] - scope-132
+| | |
+| | Less Than or Equal[boolean] - scope-135
+| | |
+| | |---Project[int][0] - scope-133
+| | |
+| | |---Constant(10) - scope-134
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-141
+| |
+| |---d: Filter[bag] - scope-137
+| | |
+| | Greater Than[boolean] - scope-140
+| | |
+| | |---Project[int][0] - scope-138
+| | |
+| | |---Constant(10) - scope-139
+|
+|---a: New For Each(false,false)[bag] - scope-125
+ | |
+ | Cast[int] - scope-120
+ | |
+ | |---Project[bytearray][0] - scope-119
+ | |
+ | Cast[int] - scope-123
+ | |
+ | |---Project[bytearray][1] - scope-122
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-118
\ No newline at end of file
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld Tue Dec 3 18:07:50 2013
@@ -0,0 +1,81 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-196
+#--------------------------------------------------
+Tez vertex scope-182
+# Plan on vertex
+7-1: Split - scope-157
+| |
+| b1: Local Rearrange[tuple]{int}(false) - scope-164
+| | |
+| | Project[int][0] - scope-165
+| |
+| |---b: Filter[bag] - scope-158
+| | |
+| | Less Than or Equal[boolean] - scope-161
+| | |
+| | |---Project[int][0] - scope-159
+| | |
+| | |---Constant(5) - scope-160
+| |
+| Local Rearrange[tuple]{tuple}(false) - scope-188
+| | |
+| | Project[tuple][*] - scope-187
+| |
+| |---c1: Limit - scope-171
+| |
+| |---c: Filter[bag] - scope-167
+| | |
+| | Less Than or Equal[boolean] - scope-170
+| | |
+| | |---Project[int][0] - scope-168
+| | |
+| | |---Constant(10) - scope-169
+| |
+| d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-181
+| |
+| |---d1: Filter[bag] - scope-177
+| | |
+| | Equal To[boolean] - scope-180
+| | |
+| | |---Project[int][0] - scope-178
+| | |
+| | |---Constant(5) - scope-179
+| |
+| |---d: Filter[bag] - scope-173
+| | |
+| | Greater Than[boolean] - scope-176
+| | |
+| | |---Project[int][0] - scope-174
+| | |
+| | |---Constant(10) - scope-175
+|
+|---a: New For Each(false,false)[bag] - scope-156
+ | |
+ | Cast[int] - scope-151
+ | |
+ | |---Project[bytearray][0] - scope-150
+ | |
+ | Cast[int] - scope-154
+ | |
+ | |---Project[bytearray][1] - scope-153
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-149
+Tez vertex scope-184
+# Plan on vertex
+b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-166
+|
+|---b1: Package(Packager)[tuple]{int} - scope-163
+Tez vertex scope-189
+# Plan on vertex
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-172
+|
+|---Limit - scope-193
+ |
+ |---New For Each(true)[bag] - scope-192
+ | |
+ | Project[tuple][1] - scope-191
+ |
+ |---Package(Packager)[tuple]{tuple} - scope-190
\ No newline at end of file
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld?rev=1547505&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld Tue Dec 3 18:07:50 2013
@@ -0,0 +1,49 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ plan: scope-227
+#--------------------------------------------------
+Tez vertex scope-221
+# Plan on vertex
+a: Split - scope-205
+| |
+| b: Local Rearrange[tuple]{int}(false) - scope-210
+| | |
+| | Project[int][0] - scope-211
+| |
+| |---a: Filter[bag] - scope-206
+| | |
+| | Constant(true) - scope-207
+| |
+| c: Local Rearrange[tuple]{tuple}(false) - scope-217
+| | |
+| | Project[int][0] - scope-218
+| | |
+| | Project[int][1] - scope-219
+| |
+| |---a: Filter[bag] - scope-213
+| | |
+| | Constant(true) - scope-214
+|
+|---a: New For Each(false,false)[bag] - scope-204
+ | |
+ | Cast[int] - scope-199
+ | |
+ | |---Project[bytearray][0] - scope-198
+ | |
+ | Cast[int] - scope-202
+ | |
+ | |---Project[bytearray][1] - scope-201
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-197
+Tez vertex scope-226
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-220
+|
+|---c: Package(Packager)[tuple]{tuple} - scope-216
+Tez vertex scope-223
+# Plan on vertex
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-212
+|
+|---b: Package(Packager)[tuple]{int} - scope-209
\ No newline at end of file
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java Tue Dec 3 18:07:50 2013
@@ -53,11 +53,14 @@ import org.junit.runner.RunWith;
*/
@RunWith(OrderedJUnit4Runner.class)
@TestOrder({
- "testRun1",
- "testRun2",
- "testRun3",
- "testRun4",
- "testRun5"
+ "testFilter",
+ "testGroupBy",
+ "testJoin",
+ "testLimit",
+ "testDistinct",
+ "testSplitSingleVertex",
+ "testSplitMultiVertex",
+ "testMultipleGroupBySplit"
})
public class TestTezCompiler {
private static PigContext pc;
@@ -79,7 +82,7 @@ public class TestTezCompiler {
}
@Test
- public void testRun1() throws Exception {
+ public void testFilter() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +
"b = filter a by x > 0;" +
@@ -91,7 +94,7 @@ public class TestTezCompiler {
}
@Test
- public void testRun2() throws Exception {
+ public void testGroupBy() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +
"b = group a by x;" +
@@ -103,7 +106,7 @@ public class TestTezCompiler {
}
@Test
- public void testRun3() throws Exception {
+ public void testJoin() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +
"b = load 'file:///tmp/input2' as (x:int, z:int);" +
@@ -116,7 +119,7 @@ public class TestTezCompiler {
}
@Test
- public void testRun4() throws Exception {
+ public void testLimit() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +
"b = limit a 10;" +
@@ -128,7 +131,7 @@ public class TestTezCompiler {
}
@Test
- public void testRun5() throws Exception {
+ public void testDistinct() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +
"b = distinct a;" +
@@ -139,6 +142,52 @@ public class TestTezCompiler {
run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld");
}
+ @Test
+ public void testSplitSingleVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';" +
+ "store d into 'file:///tmp/output/d';";
+
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld");
+ }
+
+ @Test
+ public void testSplitMultiVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "b1 = group b by x;" +
+ "c1 = limit c 1;" +
+ "d1 = filter d by x == 5;" +
+ //"d2 = order d by x;" + //TODO
+ //"d3 = join c1 by x, d1 by x;" +
+ "store b1 into 'file:///tmp/output/b1';" +
+ "store c1 into 'file:///tmp/output/c1';" +
+ "store d1 into 'file:///tmp/output/d1';";
+ //"store d2 into 'file:///tmp/output/d2';" +
+ //"store d3 into 'file:///tmp/output/d3';";
+
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld");
+ }
+
+ @Test
+ public void testMultipleGroupBySplit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "c = group a by (x,y);" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';";
+
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ run(pp, "test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld");
+ }
+
private void run(PhysicalPlan pp, String expectedFile) throws Exception {
TezCompiler comp = new TezCompiler(pp, pc);
comp.compile();
@@ -149,6 +198,8 @@ public class TestTezCompiler {
TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, tezPlanContainer);
printer.visit();
String compiledPlan = baos.toString();
+ System.out.println();
+ System.out.println("<<<" + compiledPlan + ">>>");
FileInputStream fis = new FileInputStream(expectedFile);
byte[] b = new byte[MAX_SIZE];
@@ -159,8 +210,6 @@ public class TestTezCompiler {
goldenPlan = goldenPlan.substring(0, len-1);
}
- System.out.println();
- System.out.println("<<<" + compiledPlan + ">>>");
System.out.println("-------------");
System.out.println("Golden");
System.out.println("<<<" + goldenPlan + ">>>");
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1547505&r1=1547504&r2=1547505&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezJobControlCompiler.java Tue Dec 3 18:07:50 2013
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
@@ -60,10 +61,14 @@ import org.junit.runner.RunWith;
public class TestTezJobControlCompiler {
private static PigContext pc;
private static PigServer pigServer;
+ private static URI input1;
+ private static URI input2;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
pc = new PigContext(new TezLocalExecType(), new Properties());
+ input1 = Util.createTempFileDelOnExit("input1", "txt").toURI();
+ input2 = Util.createTempFileDelOnExit("input2", "txt").toURI();
}
@AfterClass
@@ -78,7 +83,7 @@ public class TestTezJobControlCompiler {
@Test
public void testRun1() throws Exception {
String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "a = load '" + input1 +"' as (x:int, y:int);" +
"b = filter a by x > 0;" +
"c = foreach b generate y;" +
"store c into 'file:///tmp/output';";
@@ -89,7 +94,7 @@ public class TestTezJobControlCompiler {
List<TezOperator> roots = compiledPlan.first.getRoots();
assertEquals(1, roots.size());
- Vertex root = compiledPlan.second.getVertex(roots.get(0).name());
+ Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
assertNotNull(root);
assertEquals(0, root.getInputVertices().size());
assertEquals(0, root.getOutputVertices().size());
@@ -98,7 +103,7 @@ public class TestTezJobControlCompiler {
@Test
public void testRun2() throws Exception {
String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "a = load '" + input1 +"' as (x:int, y:int);" +
"b = group a by x;" +
"c = foreach b generate group, a;" +
"store c into 'file:///tmp/output';";
@@ -112,12 +117,12 @@ public class TestTezJobControlCompiler {
List<TezOperator> leaves = compiledPlan.first.getLeaves();
assertEquals(1, leaves.size());
- Vertex root = compiledPlan.second.getVertex(roots.get(0).name());
+ Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
assertNotNull(root);
assertEquals(0, root.getInputVertices().size());
assertEquals(1, root.getOutputVertices().size());
- Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).name());
+ Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
assertNotNull(leaf);
assertEquals(1, leaf.getInputVertices().size());
assertEquals(0, leaf.getOutputVertices().size());
@@ -129,8 +134,8 @@ public class TestTezJobControlCompiler {
@Test
public void testRun3() throws Exception {
String query =
- "a = load 'file:///tmp/input1' as (x:int, y:int);" +
- "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "a = load '" + input1 +"' as (x:int, y:int);" +
+ "b = load '" + input2 +"' as (x:int, z:int);" +
"c = join a by x, b by x;" +
"d = foreach c generate a::x as x, y, z;" +
"store d into 'file:///tmp/output';";
@@ -144,14 +149,14 @@ public class TestTezJobControlCompiler {
List<TezOperator> leaves = compiledPlan.first.getLeaves();
assertEquals(1, leaves.size());
- Vertex root0 = compiledPlan.second.getVertex(roots.get(0).name());
- Vertex root1 = compiledPlan.second.getVertex(roots.get(1).name());
+ Vertex root0 = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
+ Vertex root1 = compiledPlan.second.getVertex(roots.get(1).getOperatorKey().toString());
assertNotNull(root0);
assertNotNull(root1);
assertEquals(0, root0.getInputVertices().size());
assertEquals(1, root1.getOutputVertices().size());
- Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).name());
+ Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
assertNotNull(leaf);
assertEquals(2, leaf.getInputVertices().size());
assertEquals(0, leaf.getOutputVertices().size());