You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/12/05 01:23:38 UTC
svn commit: r1547973 -
/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Author: daijy
Date: Thu Dec 5 00:23:38 2013
New Revision: 1547973
URL: http://svn.apache.org/r1547973
Log:
PIG-3610: Fix e2e tests Operators_3, Operators_5, Join_1
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1547973&r1=1547972&r2=1547973&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Thu Dec 5 00:23:38 2013
@@ -196,6 +196,30 @@ public class TezDagBuilder extends TezOp
break;
}
}
+
+ conf.setBoolean("mapred.mapper.new-api", true);
+ conf.set("pig.pigContext", ObjectSerializer.serialize(pc));
+
+ if(from.isGlobalSort() || from.isLimitAfterSort()){
+ if (from.isGlobalSort()) {
+ FileSystem fs = FileSystem.get(globalConf);
+ Path quantFilePath = new Path(from.getQuantFile() + "/part-r-00000");
+ FileStatus fstat = fs.getFileStatus(quantFilePath);
+ LocalResource quantFileResource = LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
+ LocalResourceType.FILE,
+ LocalResourceVisibility.APPLICATION,
+ fstat.getLen(),
+ fstat.getModificationTime());
+ localResources.put(quantFilePath.getName(), quantFileResource);
+ conf.set("pig.quantilesFile", fstat.getPath().toString());
+ conf.set("pig.sortOrder",
+ ObjectSerializer.serialize(from.getSortOrder()));
+ conf.setClass("mapreduce.job.partitioner.class",
+ WeightedRangePartitioner.class,
+ Partitioner.class);
+ }
+ }
in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
out.setUserPayload(TezUtils.createUserPayloadFromConf(conf));
@@ -295,27 +319,6 @@ public class TezDagBuilder extends TezOp
selectInputComparator(payloadConf, pack.getPkgr().getKeyType());
}
- if(tezOp.isGlobalSort() || tezOp.isLimitAfterSort()){
- if (tezOp.isGlobalSort()) {
- FileSystem fs = FileSystem.get(globalConf);
- Path quantFilePath = new Path(tezOp.getQuantFile() + "/part-r-00000");
- FileStatus fstat = fs.getFileStatus(quantFilePath);
- LocalResource quantFileResource = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
- LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION,
- fstat.getLen(),
- fstat.getModificationTime());
- localResources.put(quantFilePath.getName(), quantFileResource);
- payloadConf.set("pig.quantilesFile", fstat.getPath().toString());
- payloadConf.set("pig.sortOrder",
- ObjectSerializer.serialize(tezOp.getSortOrder()));
- payloadConf.setClass("mapreduce.job.partitioner.class",
- WeightedRangePartitioner.class,
- Partitioner.class);
- }
- }
-
payloadConf.setClass("mapreduce.outputformat.class",
PigOutputFormat.class, OutputFormat.class);