You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2014/03/24 19:44:34 UTC
svn commit: r1580988 - in /hadoop/common/branches/branch-1-win:
CHANGES.branch-1-win.txt
src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
Author: cnauroth
Date: Mon Mar 24 18:44:34 2014
New Revision: 1580988
URL: http://svn.apache.org/r1580988
Log:
MAPREDUCE-5808. Port output replication factor configurable for terasort to Hadoop 1.x. Contributed by Chuan Liu.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1580988&r1=1580987&r2=1580988&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Mon Mar 24 18:44:34 2014
@@ -513,3 +513,6 @@ Branch-hadoop-1-win (branched from branc
HDFS-5516. WebHDFS does not require user name when anonymous http requests
are disallowed. (Miodrag Radulovic via cnauroth)
+
+ MAPREDUCE-5808. Port output replication factor configurable for terasort to
+ Hadoop 1.x. (Chuan Liu via cnauroth)
Modified: hadoop/common/branches/branch-1-win/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1580988&r1=1580987&r2=1580988&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java (original)
+++ hadoop/common/branches/branch-1-win/src/examples/org/apache/hadoop/examples/terasort/TeraSort.java Mon Mar 24 18:44:34 2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.util.ToolRunner
*/
public class TeraSort extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(TeraSort.class);
+ private static final String OUTPUT_REPLICATION = "mapreduce.terasort.output.replication";
/**
* A partitioner that splits text keys into roughly equal partitions
@@ -222,6 +223,10 @@ public class TeraSort extends Configured
}
}
+
+ public static int getOutputReplication(JobConf job) {
+ return job.getInt(OUTPUT_REPLICATION, 1);
+ }
public int run(String[] args) throws Exception {
LOG.info("starting");
@@ -243,7 +248,7 @@ public class TeraSort extends Configured
TeraInputFormat.writePartitionFile(job, partitionFile);
DistributedCache.addCacheFile(partitionUri, job);
DistributedCache.createSymlink(job);
- job.setInt("dfs.replication", 1);
+ job.setInt("dfs.replication", getOutputReplication(job));
TeraOutputFormat.setFinalSync(job, true);
JobClient.runJob(job);
LOG.info("done");