You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/09/29 02:35:18 UTC
[12/50] [abbrv] git commit: TEZ-1575. MRRSleepJob does not pick MR
settings for container size and java opts. (hitesh)
TEZ-1575. MRRSleepJob does not pick MR settings for container size and java opts. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b62298e1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b62298e1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b62298e1
Branch: refs/heads/branch-0.5
Commit: b62298e1054c934e4ee8909b7ba3165ce344ea8b
Parents: d1ea44d
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Sep 12 11:02:51 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Sep 12 11:02:51 2014 -0700
----------------------------------------------------------------------
.../tez/mapreduce/examples/MRRSleepJob.java | 25 +++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b62298e1/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 3f9d222..de8a1ad 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -525,9 +525,11 @@ public class MRRSleepJob extends Configured implements Tool {
int numTasks = generateSplitsInAM ? -1 : numMapper;
Vertex mapVertex = Vertex.create("map", ProcessorDescriptor.create(
- MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
- .addTaskLocalFiles(commonLocalResources);
- mapVertex.addDataSource("MRInput", dataSource);
+ MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks,
+ MRHelpers.getResourceForMRMapper(mapStageConf));
+ mapVertex.addTaskLocalFiles(commonLocalResources)
+ .addDataSource("MRInput", dataSource)
+ .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRMapper(mapStageConf));
vertices.add(mapVertex);
if (iReduceStagesCount > 0
@@ -538,8 +540,11 @@ public class MRRSleepJob extends Configured implements Tool {
UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
Vertex ivertex = Vertex.create("ireduce" + (i + 1),
ProcessorDescriptor.create(ReduceProcessor.class.getName()).
- setUserPayload(iReduceUserPayload), numIReducer);
- ivertex.addTaskLocalFiles(commonLocalResources);
+ setUserPayload(iReduceUserPayload), numIReducer,
+ MRHelpers.getResourceForMRReducer(intermediateReduceStageConfs[i]));
+ ivertex.addTaskLocalFiles(commonLocalResources)
+ .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(
+ intermediateReduceStageConfs[i]));
vertices.add(ivertex);
}
}
@@ -548,10 +553,12 @@ public class MRRSleepJob extends Configured implements Tool {
if (numReducer > 0) {
UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
finalReduceVertex = Vertex.create("reduce", ProcessorDescriptor.create(
- ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
- finalReduceVertex.addTaskLocalFiles(commonLocalResources);
- finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigBuilder(finalReduceConf,
- NullOutputFormat.class).build());
+ ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer,
+ MRHelpers.getResourceForMRReducer(finalReduceConf));
+ finalReduceVertex.addTaskLocalFiles(commonLocalResources)
+ .addDataSink("MROutput", MROutputLegacy.createConfigBuilder(
+ finalReduceConf, NullOutputFormat.class).build())
+ .setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(finalReduceConf));
vertices.add(finalReduceVertex);
} else {
// Map only job