You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/12/30 22:14:46 UTC
svn commit: r1554320 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/had...
Author: vinodkv
Date: Mon Dec 30 21:14:46 2013
New Revision: 1554320
URL: http://svn.apache.org/r1554320
Log:
MAPREDUCE-5685. Fixed a bug with JobContext getCacheFiles API inside the WrappedReducer class. Contributed by Yi Song.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1554320&r1=1554319&r2=1554320&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Dec 30 21:14:46 2013
@@ -264,6 +264,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5694. Fixed MR AppMaster to shutdown the LogManager so as to avoid
losing syslog in some conditions. (Mohammad Kamrul Islam via vinodkv)
+ MAPREDUCE-5685. Fixed a bug with JobContext getCacheFiles API inside the
+ WrappedReducer class. (Yi Song via vinodkv)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java?rev=1554320&r1=1554319&r2=1554320&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Mon Dec 30 21:14:46 2013
@@ -44,6 +44,8 @@ import org.apache.hadoop.io.NullWritable
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
@@ -82,12 +84,11 @@ public class TestMRWithDistributedCache
private static final Log LOG =
LogFactory.getLog(TestMRWithDistributedCache.class);
+
+ private static class DistributedCacheChecker {
- public static class DistributedCacheChecker extends
- Mapper<LongWritable, Text, NullWritable, NullWritable> {
-
- @Override
- public void setup(Context context) throws IOException {
+ public void setup(TaskInputOutputContext<?, ?, ?, ?> context)
+ throws IOException {
Configuration conf = context.getConfiguration();
Path[] localFiles = context.getLocalCacheFiles();
URI[] files = context.getCacheFiles();
@@ -101,6 +102,10 @@ public class TestMRWithDistributedCache
TestCase.assertEquals(2, files.length);
TestCase.assertEquals(2, archives.length);
+ // Check the file name
+ TestCase.assertTrue(files[0].getPath().endsWith("distributed.first"));
+ TestCase.assertTrue(files[1].getPath().endsWith("distributed.second.jar"));
+
// Check lengths of the files
TestCase.assertEquals(1, fs.getFileStatus(localFiles[0]).getLen());
TestCase.assertTrue(fs.getFileStatus(localFiles[1]).getLen() > 1);
@@ -130,8 +135,28 @@ public class TestMRWithDistributedCache
TestCase.assertTrue("second file should be symlinked too",
expectedAbsentSymlinkFile.exists());
}
+
}
-
+
+ public static class DistributedCacheCheckerMapper extends
+ Mapper<LongWritable, Text, NullWritable, NullWritable> {
+
+ @Override
+ protected void setup(Context context) throws IOException,
+ InterruptedException {
+ new DistributedCacheChecker().setup(context);
+ }
+ }
+
+ public static class DistributedCacheCheckerReducer extends
+ Reducer<LongWritable, Text, NullWritable, NullWritable> {
+
+ @Override
+ public void setup(Context context) throws IOException {
+ new DistributedCacheChecker().setup(context);
+ }
+ }
+
private void testWithConf(Configuration conf) throws IOException,
InterruptedException, ClassNotFoundException, URISyntaxException {
// Create a temporary file of length 1.
@@ -146,7 +171,8 @@ public class TestMRWithDistributedCache
Job job = Job.getInstance(conf);
- job.setMapperClass(DistributedCacheChecker.class);
+ job.setMapperClass(DistributedCacheCheckerMapper.class);
+ job.setReducerClass(DistributedCacheCheckerReducer.class);
job.setOutputFormatClass(NullOutputFormat.class);
FileInputFormat.setInputPaths(job, first);
// Creates the Job Configuration
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java?rev=1554320&r1=1554319&r2=1554320&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/reduce/WrappedReducer.java Mon Dec 30 21:14:46 2013
@@ -137,7 +137,7 @@ public class WrappedReducer<KEYIN, VALUE
@Override
public URI[] getCacheFiles() throws IOException {
- return reduceContext.getCacheArchives();
+ return reduceContext.getCacheFiles();
}
@Override