You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/04/10 01:25:25 UTC
svn commit: r1311518 - in
/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ assembly/
conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/
hadoop-mapreduce-client/hadoop-mapreduce-client-cor...
Author: szetszwo
Date: Mon Apr 9 23:25:17 2012
New Revision: 1311518
URL: http://svn.apache.org/viewvc?rev=1311518&view=rev
Log:
Merge r1310141 through r1311517 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
- copied unchanged from r1311517, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
Removed:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/assembly/all.xml
Modified:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMRNotification.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1310141-1311517
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt Mon Apr 9 23:25:17 2012
@@ -204,7 +204,20 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4097. tools testcases fail because missing mrapp-generated-classpath
file in classpath (rvs via tucu)
-
+
+ MAPREDUCE-4113. Fix tests org.apache.hadoop.mapred.TestClusterMRNotification
+ (Devaraj K via bobby)
+
+ MAPREDUCE-4112. Fix tests org.apache.hadoop.mapred.TestClusterMapReduceTestCase
+ (Devaraj K via bobby)
+
+ MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
+ K via bobby)
+
+ MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
+ org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
+ bobby)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -264,6 +277,12 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4073. CS assigns multiple off-switch containers when using
multi-level-queues (Siddharth Seth via bobby)
+ MAPREDUCE-4051. Remove the empty hadoop-mapreduce-project/assembly/all.xml
+ file (Ravi Prakash via bobby)
+
+ MAPREDUCE-4117. mapred job -status throws NullPointerException (Devaraj K
+ via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1310141-1311517
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java Mon Apr 9 23:25:17 2012
@@ -509,6 +509,11 @@ public class Job extends JobContextImpl
lastEvent = event;
}
}
+ if (lastEvent == null) {
+ return "There are no failed tasks for the job. "
+ + "Job is failed due to some other reason and reason "
+ + "can be found in the logs.";
+ }
String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2);
String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length()-2);
return (" task " + taskID + " failed " +
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1310141-1311517
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMRNotification.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMRNotification.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMRNotification.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMRNotification.java Mon Apr 9 23:25:17 2012
@@ -20,12 +20,9 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
-import org.junit.Ignore;
-
/**
* Tests Job end notification in cluster mode.
*/
-@Ignore
public class TestClusterMRNotification extends NotificationTestCase {
public TestClusterMRNotification() throws IOException {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClusterMapReduceTestCase.java Mon Apr 9 23:25:17 2012
@@ -17,15 +17,18 @@
*/
package org.apache.hadoop.mapred;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Properties;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.junit.Ignore;
-
-import java.io.*;
-import java.util.Properties;
-@Ignore
public class TestClusterMapReduceTestCase extends ClusterMapReduceTestCase {
public void _testMapReduce(boolean restart) throws Exception {
OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt"));
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobName.java Mon Apr 9 23:25:17 2012
@@ -18,23 +18,17 @@
package org.apache.hadoop.mapred;
import java.io.BufferedReader;
-import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.util.Iterator;
-import java.util.StringTokenizer;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.serializer.JavaSerializationComparator;
import org.apache.hadoop.mapred.lib.IdentityMapper;
-import org.junit.Ignore;
-@Ignore
public class TestJobName extends ClusterMapReduceTestCase {
public void testComplexName() throws Exception {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java Mon Apr 9 23:25:17 2012
@@ -18,47 +18,43 @@
package org.apache.hadoop.mapred;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URI;
import junit.framework.TestCase;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.junit.Ignore;
+import org.junit.Assert;
+import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with multiple directories
* and check for correct classpath
*/
-@Ignore
-public class TestMiniMRClasspath extends TestCase {
+public class TestMiniMRClasspath {
- static void configureWordCount(FileSystem fs,
- String jobTracker,
- JobConf conf,
- String input,
- int numMaps,
- int numReduces,
- Path inDir, Path outDir) throws IOException {
+ static void configureWordCount(FileSystem fs, JobConf conf, String input,
+ int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
fs.delete(outDir, true);
if (!fs.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
- {
- DataOutputStream file = fs.create(new Path(inDir, "part-0"));
- file.writeBytes(input);
- file.close();
- }
+ DataOutputStream file = fs.create(new Path(inDir, "part-0"));
+ file.writeBytes(input);
+ file.close();
FileSystem.setDefaultUri(conf, fs.getUri());
- conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
- conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+ conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@@ -74,18 +70,17 @@ public class TestMiniMRClasspath extends
FileOutputFormat.setOutputPath(conf, outDir);
conf.setNumMapTasks(numMaps);
conf.setNumReduceTasks(numReduces);
- //pass a job.jar already included in the hadoop build
- conf.setJar("build/test/mapred/testjar/testjob.jar");
+ //set the tests jar file
+ conf.setJarByClass(TestMiniMRClasspath.class);
}
- static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
- String input, int numMaps, int numReduces)
+ static String launchWordCount(URI fileSys, JobConf conf, String input,
+ int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/wc/input");
final Path outDir = new Path("/testing/wc/output");
FileSystem fs = FileSystem.get(fileSys, conf);
- configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir,
- outDir);
+ configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
{
@@ -107,8 +102,8 @@ public class TestMiniMRClasspath extends
return result.toString();
}
- static String launchExternal(URI uri, String jobTracker, JobConf conf,
- String input, int numMaps, int numReduces)
+ static String launchExternal(URI uri, JobConf conf, String input,
+ int numMaps, int numReduces)
throws IOException {
final Path inDir = new Path("/testing/ext/input");
@@ -124,8 +119,7 @@ public class TestMiniMRClasspath extends
file.close();
}
FileSystem.setDefaultUri(conf, uri);
- conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
- conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
+ conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
conf.setJobName("wordcount");
conf.setInputFormat(TextInputFormat.class);
@@ -142,8 +136,8 @@ public class TestMiniMRClasspath extends
conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer");
conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
- //pass a job.jar already included in the hadoop build
- conf.setJar("build/test/mapred/testjar/testjob.jar");
+ // set the tests jar file
+ conf.setJarByClass(TestMiniMRClasspath.class);
JobClient.runJob(conf);
StringBuffer result = new StringBuffer();
Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
@@ -164,6 +158,7 @@ public class TestMiniMRClasspath extends
return result.toString();
}
+ @Test
public void testClassPath() throws IOException {
String namenode = null;
MiniDFSCluster dfs = null;
@@ -180,13 +175,10 @@ public class TestMiniMRClasspath extends
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
- final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
- result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf,
- "The quick brown fox\nhas many silly\n" +
- "red fox sox\n",
- 3, 1);
- assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
- "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
+ result = launchWordCount(fileSys.getUri(), jobConf,
+ "The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
+ Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
+ + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
} finally {
if (dfs != null) { dfs.shutdown(); }
@@ -195,6 +187,7 @@ public class TestMiniMRClasspath extends
}
}
+ @Test
public void testExternalWritable()
throws IOException {
@@ -214,12 +207,10 @@ public class TestMiniMRClasspath extends
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = new JobConf();
String result;
- final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
- result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf,
- "Dennis was here!\nDennis again!",
- 3, 1);
- assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
+ result = launchExternal(fileSys.getUri(), jobConf,
+ "Dennis was here!\nDennis again!", 3, 1);
+ Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
}
finally {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java Mon Apr 9 23:25:17 2012
@@ -17,26 +17,26 @@
*/
package org.apache.hadoop.mapred;
-import java.io.*;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.security.*;
-import org.junit.Ignore;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
*/
-@Ignore
-public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
+public class TestMiniMRWithDFSWithDistinctUsers {
static final UserGroupInformation DFS_UGI = createUGI("dfs", true);
static final UserGroupInformation ALICE_UGI = createUGI("alice", false);
static final UserGroupInformation BOB_UGI = createUGI("bob", false);
@@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistin
MiniDFSCluster dfs = null;
FileSystem fs = null;
Configuration conf = new Configuration();
- String jobTrackerName;
static UserGroupInformation createUGI(String name, boolean issuper) {
String group = issuper? "supergroup": name;
@@ -71,9 +70,10 @@ public class TestMiniMRWithDFSWithDistin
});
rj.waitForCompletion();
- assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
+ Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
}
+ @Before
public void setUp() throws Exception {
dfs = new MiniDFSCluster(conf, 4, true, null);
@@ -96,29 +96,30 @@ public class TestMiniMRWithDFSWithDistin
mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
1, null, null, MR_UGI, mrConf);
- jobTrackerName = "localhost:" + mr.getJobTrackerPort();
}
+ @After
public void tearDown() throws Exception {
if (mr != null) { mr.shutdown();}
if (dfs != null) { dfs.shutdown(); }
}
+ @Test
public void testDistinctUsers() throws Exception {
JobConf job1 = mr.createJobConf();
String input = "The quick brown fox\nhas many silly\n"
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
- input, 2, 1, inDir, outDir);
+ TestMiniMRClasspath
+ .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
JobConf job2 = mr.createJobConf();
Path inDir2 = new Path("/testing/distinct/input2");
Path outDir2 = new Path("/user/bob/output2");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2,
- input, 2, 1, inDir2, outDir2);
+ TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
+ outDir2);
runJobAsUser(job2, BOB_UGI);
}
@@ -127,6 +128,7 @@ public class TestMiniMRWithDFSWithDistin
* task makes lots of spills (more than fit in the spill index cache)
* that it will succeed.
*/
+ @Test
public void testMultipleSpills() throws Exception {
JobConf job1 = mr.createJobConf();
@@ -141,8 +143,8 @@ public class TestMiniMRWithDFSWithDistin
+ "red fox sox\n";
Path inDir = new Path("/testing/distinct/input");
Path outDir = new Path("/user/alice/output");
- TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
- input, 2, 1, inDir, outDir);
+ TestMiniMRClasspath
+ .configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
runJobAsUser(job1, ALICE_UGI);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java?rev=1311518&r1=1311517&r2=1311518&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerTokenPBImpl.java Mon Apr 9 23:25:17 2012
@@ -155,4 +155,12 @@ public class ContainerTokenPBImpl extend
builder.setService((service));
}
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("ContainerToken { ");
+ sb.append("kind: ").append(getKind()).append(", ");
+ sb.append("service: ").append(getService()).append(" }");
+ return sb.toString();
+ }
}
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1310141-1311517
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1310141-1311517