You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2015/03/21 21:50:43 UTC
hadoop git commit: MAPREDUCE-6239. Consolidate TestJobConf classes in
hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core.
Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9c494ceda -> 97da36777
MAPREDUCE-6239. Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97da3677
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97da3677
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97da3677
Branch: refs/heads/branch-2
Commit: 97da36777f2916904e44bfa011208f4f07fe5170
Parents: 9c494ce
Author: Harsh J <ha...@cloudera.com>
Authored: Sun Mar 22 02:20:11 2015 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Sun Mar 22 02:20:11 2015 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 +
.../org/apache/hadoop/mapred/TestJobConf.java | 171 ++++++++++++++++
.../org/apache/hadoop/conf/TestJobConf.java | 198 -------------------
3 files changed, 175 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97da3677/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0ffbc28..da43195 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -8,6 +8,10 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-6239. Consolidate TestJobConf classes in
+ hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core
+ (Varun Saxena via harsh)
+
MAPREDUCE-5807. Print usage by TeraSort job. (Rohith via harsh)
MAPREDUCE-4653. TestRandomAlgorithm has an unused "import" statement.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97da3677/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
index 3d924e1..a68ba4f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
@@ -22,6 +22,7 @@ import java.util.regex.Pattern;
import static org.junit.Assert.*;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Assert;
import org.junit.Test;
@@ -188,4 +189,174 @@ public class TestJobConf {
Assert.assertEquals(2048, configuration.getLong(
JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
}
+
+ @Test
+ public void testProfileParamsDefaults() {
+ JobConf configuration = new JobConf();
+ String result = configuration.getProfileParams();
+ Assert.assertNotNull(result);
+ Assert.assertTrue(result.contains("file=%s"));
+ Assert.assertTrue(result.startsWith("-agentlib:hprof"));
+ }
+
+ @Test
+ public void testProfileParamsSetter() {
+ JobConf configuration = new JobConf();
+
+ configuration.setProfileParams("test");
+ Assert.assertEquals("test", configuration.get(MRJobConfig.TASK_PROFILE_PARAMS));
+ }
+
+ @Test
+ public void testProfileParamsGetter() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(MRJobConfig.TASK_PROFILE_PARAMS, "test");
+ Assert.assertEquals("test", configuration.getProfileParams());
+ }
+
+ /**
+ * Testing mapred.task.maxvmem replacement with new values
+ *
+ */
+ @Test
+ public void testMemoryConfigForMapOrReduceTask(){
+ JobConf configuration = new JobConf();
+ configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),300);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),300);
+
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , "-1");
+ configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(400));
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
+ configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
+ Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, "3");
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+ }
+
+ /**
+ * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
+ * new configuration keys' values to be used.
+ */
+ @Test
+ public void testNegativeValueForTaskVmem() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
+ Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
+ configuration.getMemoryForMapTask());
+ Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
+ configuration.getMemoryForReduceTask());
+
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, "4");
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "5");
+ Assert.assertEquals(4, configuration.getMemoryForMapTask());
+ Assert.assertEquals(5, configuration.getMemoryForReduceTask());
+
+ }
+
+ /**
+ * Test that negative values for new configuration keys get passed through.
+ */
+ @Test
+ public void testNegativeValuesForMemoryParams() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
+ Assert.assertEquals(-5, configuration.getMemoryForMapTask());
+ Assert.assertEquals(-6, configuration.getMemoryForReduceTask());
+ }
+
+ /**
+ * Test deprecated accessor and mutator method for mapred.task.maxvmem
+ */
+ @Test
+ public void testMaxVirtualMemoryForTask() {
+ JobConf configuration = new JobConf();
+
+ //get test case
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
+ configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+ //set test case
+
+ configuration = new JobConf();
+ configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+
+ configuration = new JobConf();
+ configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
+ configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(400));
+ configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+
+
+ }
+
+ /**
+ * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
+ * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
+ * failed tasks will be retried on other nodes
+ */
+ @Test
+ public void testMaxTaskFailuresPerTracker() {
+ JobConf jobConf = new JobConf(true);
+ Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
+ + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
+ ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
+ jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97da3677/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
deleted file mode 100644
index e380d92..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.conf;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-
-public class TestJobConf {
-
- @Test
- public void testProfileParamsDefaults() {
- JobConf configuration = new JobConf();
- String result = configuration.getProfileParams();
- Assert.assertNotNull(result);
- Assert.assertTrue(result.contains("file=%s"));
- Assert.assertTrue(result.startsWith("-agentlib:hprof"));
- }
-
- @Test
- public void testProfileParamsSetter() {
- JobConf configuration = new JobConf();
-
- configuration.setProfileParams("test");
- Assert.assertEquals("test", configuration.get(MRJobConfig.TASK_PROFILE_PARAMS));
- }
-
- @Test
- public void testProfileParamsGetter() {
- JobConf configuration = new JobConf();
-
- configuration.set(MRJobConfig.TASK_PROFILE_PARAMS, "test");
- Assert.assertEquals("test", configuration.getProfileParams());
- }
-
- /**
- * Testing mapred.task.maxvmem replacement with new values
- *
- */
- @Test
- public void testMemoryConfigForMapOrReduceTask(){
- JobConf configuration = new JobConf();
- configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
- Assert.assertEquals(configuration.getMemoryForMapTask(),300);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),300);
-
- configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
- configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(300));
- Assert.assertEquals(configuration.getMemoryForMapTask(),2);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
- configuration = new JobConf();
- configuration.set("mapred.task.maxvmem" , "-1");
- configuration.set(MRJobConfig.MAP_MEMORY_MB,String.valueOf(300));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB,String.valueOf(400));
- Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
- Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
-
- configuration = new JobConf();
- configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
- configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
- Assert.assertEquals(configuration.getMemoryForMapTask(),2);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
- configuration = new JobConf();
- configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
- configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
- Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
-
- configuration = new JobConf();
- configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
- configuration.set(MRJobConfig.MAP_MEMORY_MB, "3");
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
- Assert.assertEquals(configuration.getMemoryForMapTask(),2);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
- }
-
- /**
- * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
- * new configuration keys' values to be used.
- */
- @Test
- public void testNegativeValueForTaskVmem() {
- JobConf configuration = new JobConf();
-
- configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
- Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
- configuration.getMemoryForMapTask());
- Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
- configuration.getMemoryForReduceTask());
-
- configuration.set(MRJobConfig.MAP_MEMORY_MB, "4");
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "5");
- Assert.assertEquals(4, configuration.getMemoryForMapTask());
- Assert.assertEquals(5, configuration.getMemoryForReduceTask());
-
- }
-
- /**
- * Test that negative values for new configuration keys get passed through.
- */
- @Test
- public void testNegativeValuesForMemoryParams() {
- JobConf configuration = new JobConf();
-
- configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
- Assert.assertEquals(-5, configuration.getMemoryForMapTask());
- Assert.assertEquals(-6, configuration.getMemoryForReduceTask());
- }
-
- /**
- * Test deprecated accessor and mutator method for mapred.task.maxvmem
- */
- @Test
- public void testMaxVirtualMemoryForTask() {
- JobConf configuration = new JobConf();
-
- //get test case
- configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
- Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);
-
- configuration = new JobConf();
- configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
- Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);
-
- configuration = new JobConf();
- configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
- configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
- Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
-
- configuration = new JobConf();
- configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
- Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
-
- //set test case
-
- configuration = new JobConf();
- configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
- Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
- Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
-
- configuration = new JobConf();
- configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
- configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(400));
- configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
- Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
- Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
-
-
- }
-
- /**
- * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
- * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
- * failed tasks will be retried on other nodes
- */
- @Test
- public void testMaxTaskFailuresPerTracker() {
- JobConf jobConf = new JobConf(true);
- Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
- + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
- ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
- jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
- );
- }
-}