You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tu...@apache.org on 2013/07/11 02:07:42 UTC
svn commit: r1502063 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project: CHANGES.txt
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
Author: tucu
Date: Thu Jul 11 00:07:42 2013
New Revision: 1502063
URL: http://svn.apache.org/r1502063
Log:
MAPREDUCE-5333. Add test that verifies MRAM works correctly when sending requests with non-normalized capabilities. (ywskycn via tucu)
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1502063&r1=1502062&r2=1502063&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt Thu Jul 11 00:07:42 2013
@@ -171,6 +171,9 @@ Release 2.1.0-beta - 2013-07-02
MAPREDUCE-5326. Added version to shuffle header. (Zhijie Shen via
acmurthy)
+ MAPREDUCE-5333. Add test that verifies MRAM works correctly when sending
+ requests with non-normalized capabilities. (ywskycn via tucu)
+
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
Added: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java?rev=1502063&view=auto
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java (added)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java Thu Jul 11 00:07:42 2013
@@ -0,0 +1,121 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.SleepJob;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMRAMWithNonNormalizedCapabilities {
+ private static final Log LOG = LogFactory.getLog(TestMRAMWithNonNormalizedCapabilities.class);
+ private static FileSystem localFs;
+ protected static MiniMRYarnCluster mrCluster = null;
+
+ private static Configuration conf = new Configuration();
+
+ static {
+ try {
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException io) {
+ throw new RuntimeException("problem getting local fs", io);
+ }
+ }
+
+ private static Path TEST_ROOT_DIR = new Path("target",
+ TestMRAMWithNonNormalizedCapabilities.class.getName() + "-tmpDir")
+ .makeQualified(localFs.getUri(), localFs.getWorkingDirectory());
+ static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
+
+ @Before
+ public void setup() throws Exception {
+ if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ if (mrCluster == null) {
+ mrCluster = new MiniMRYarnCluster(getClass().getName());
+ mrCluster.init(new Configuration());
+ mrCluster.start();
+ }
+ // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
+ // workaround the absent public discache.
+ localFs.copyFromLocalFile(new Path(MiniMRYarnCluster.APPJAR), APP_JAR);
+ localFs.setPermission(APP_JAR, new FsPermission("700"));
+ }
+
+ /**
+ * To ensure nothing broken after we removed normalization
+ * from the MRAM side
+ * @throws Exception
+ */
+ @Test
+ public void testJobWithNonNormalizedCapabilities() throws Exception {
+ if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ JobConf jobConf = new JobConf(mrCluster.getConfig());
+ jobConf.setInt("mapreduce.map.memory.mb", 700);
+ jobConf.setInt("mapred.reduce.memory.mb", 1500);
+
+ SleepJob sleepJob = new SleepJob();
+ sleepJob.setConf(jobConf);
+ Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
+ job.setJarByClass(SleepJob.class);
+ job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
+ job.submit();
+ boolean completed = job.waitForCompletion(true);
+ Assert.assertTrue("Job should be completed", completed);
+ Assert.assertEquals("Job should be finished successfully",
+ JobStatus.State.SUCCEEDED, job.getJobState());
+ }
+
+ @After
+ public void tearDown() {
+ if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
+ LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ + " not found. Not running test.");
+ return;
+ }
+
+ if (mrCluster != null) {
+ mrCluster.stop();
+ }
+ }
+}
\ No newline at end of file