You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/04/06 20:57:44 UTC
svn commit: r1310547 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/
Author: bobby
Date: Fri Apr 6 18:57:44 2012
New Revision: 1310547
URL: http://svn.apache.org/viewvc?rev=1310547&view=rev
Log:
svn merge -c 1310546 FIXES: MAPREDUCE-4110. Fix tests in TestMiniMRClasspath & TestMiniMRWithDFSWithDistinctUsers (Devaraj K via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Apr 6 18:57:44 2012
@@ -114,7 +114,11 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
K via bobby)
-
+
+ MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
+ org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
+ bobby)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java Fri Apr 6 18:57:44 2012
@@ -18,47 +18,43 @@
package org.apache.hadoop.mapred;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URI;
import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.junit.Ignore;
+import org.junit.Assert;
+import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with multiple directories
* and check for correct classpath
*/
-@Ignore
-public class TestMiniMRClasspath extends TestCase {
+public class TestMiniMRClasspath {
- static void configureWordCount(FileSystem fs,
- String jobTracker,
- JobConf conf,
- String input,
- int numMaps,
- int numReduces,
- Path inDir, Path outDir) throws IOException {
+ static void configureWordCount(FileSystem fs, JobConf conf, String input,
+ int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
fs.delete(outDir, true);
if (!fs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
- {
- DataOutputStream file = fs.create(new Path(inDir, "part-0"));
- file.writeBytes(input);
- file.close();
- }
+ DataOutputStream file = fs.create(new Path(inDir, "part-0"));
+ file.writeBytes(input);
+ file.close();
FileSystem.setDefaultUri(conf, fs.getUri());
- conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
- conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+ conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@@ -74,18 +70,17 @@ public class TestMiniMRClasspath extends
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
- //pass a job.jar already included in the hadoop build
- conf.setJar("build/test/mapred/testjar/testjob.jar");
+ //set the tests jar file
+ conf.setJarByClass(TestMiniMRClasspath.class);
}
- static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
- String input, int numMaps, int numReduces)
+ static String launchWordCount(URI fileSys, JobConf conf, String input,
+ int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/wc/input");
final Path outDir = new Path("/testing/wc/output");
FileSystem fs = FileSystem.get(fileSys, conf);
- configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir,
- outDir);
+ configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
{
@@ -107,8 +102,8 @@ public class TestMiniMRClasspath extends
return result.toString();
}
- static String launchExternal(URI uri, String jobTracker, JobConf conf,
- String input, int numMaps, int numReduces)
+ static String launchExternal(URI uri, JobConf conf, String input,
+ int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/ext/input");
@@ -124,8 +119,7 @@ public class TestMiniMRClasspath extends
file.close();
}
FileSystem.setDefaultUri(conf, uri);
- conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
- conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+ conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@@ -142,8 +136,8 @@ public class TestMiniMRClasspath extends
conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer");
conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
- //pass a job.jar already included in the hadoop build
- conf.setJar("build/test/mapred/testjar/testjob.jar");
+ // set the tests jar file
+ conf.setJarByClass(TestMiniMRClasspath.class);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
@@ -164,6 +158,7 @@ public class TestMiniMRClasspath extends
return result.toString();
}
+ @Test
public void testClassPath() throws IOException {
String namenode = null;
MiniDFSCluster dfs = null;
@@ -180,13 +175,10 @@ public class TestMiniMRClasspath extends
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
- final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
- result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf,
- "The quick brown fox\nhas many silly\n" +
- "red fox sox\n",
- 3, 1);
- assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
- "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
+ result = launchWordCount(fileSys.getUri(), jobConf,
+ "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
+ Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
+ + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
} finally {
if (dfs != null) { dfs.shutdown(); }
@@ -195,6 +187,7 @@ public class TestMiniMRClasspath extends
}
}
+ @Test
public void testExternalWritable()
throws IOException {
@@ -214,12 +207,10 @@ public class TestMiniMRClasspath extends
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
- final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
- result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf,
- "Dennis was here!\nDennis again!",
- 3, 1);
- assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
+ result = launchExternal(fileSys.getUri(), jobConf,
+ "Dennis was here!\nDennis again!", 3, 1);
+ Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
}
finally {
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=1310547&r1=1310546&r2=1310547&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java Fri Apr 6 18:57:44 2012
@@ -17,26 +17,26 @@
*/
package org.apache.hadoop.mapred;
-import java.io.*;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.security.*;
-import org.junit.Ignore;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
*/
-@Ignore
-public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
+public class TestMiniMRWithDFSWithDistinctUsers {
static final UserGroupInformation DFS_UGI = createUGI("dfs", true);
static final UserGroupInformation ALICE_UGI = createUGI("alice", false);
static final UserGroupInformation BOB_UGI = createUGI("bob", false);
@@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistin
MiniDFSCluster dfs = null;
FileSystem fs = null;
Configuration conf = new Configuration();
- String jobTrackerName;
static UserGroupInformation createUGI(String name, boolean issuper) {
String group = issuper? "supergroup": name;
@@ -71,9 +70,10 @@ public class TestMiniMRWithDFSWithDistin
});
rj.waitForCompletion();
- assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
+ Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
}
+ @Before
public void setUp() throws Exception {
dfs = new MiniDFSCluster(conf, 4, true, null);
@@ -96,29 +96,30 @@ public class TestMiniMRWithDFSWithDistin
mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
1, null, null, MR_UGI, mrConf);
- jobTrackerName = "localhost:" + mr.getJobTrackerPort();
}
+ @After
public void tearDown() throws Exception {
if (mr != null) { mr.shutdown();}
if (dfs != null) { dfs.shutdown(); }
}
+ @Test
public void testDistinctUsers() throws Exception {
JobConf job1 = mr.createJobConf();
String input = "The quick brown fox\nhas many silly\n"
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
- input, 2, 1, inDir, outDir);
+ TestMiniMRClasspath
+ .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
JobConf job2 = mr.createJobConf();
Path inDir2 = new Path("/testing/distinct/input2");
Path outDir2 = new Path("/user/bob/output2");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2,
- input, 2, 1, inDir2, outDir2);
+ TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
+ outDir2);
runJobAsUser(job2, BOB_UGI);
}
@@ -127,6 +128,7 @@ public class TestMiniMRWithDFSWithDistin
* task makes lots of spills (more than fit in the spill index cache)
* that it will succeed.
*/
+ @Test
public void testMultipleSpills() throws Exception {
JobConf job1 = mr.createJobConf();
@@ -141,8 +143,8 @@ public class TestMiniMRWithDFSWithDistin
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
- input, 2, 1, inDir, outDir);
+ TestMiniMRClasspath
+ .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
}
}