You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2024/05/17 13:51:56 UTC
(hadoop) branch trunk updated: MAPREDUCE-7475. Fix non-idempotent unit tests (#6785)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 41eacf4914ff MAPREDUCE-7475. Fix non-idempotent unit tests (#6785)
41eacf4914ff is described below
commit 41eacf4914ffc6129f293440a9bcc99723ef3e64
Author: Kaiyao Ke <47...@users.noreply.github.com>
AuthorDate: Fri May 17 08:51:47 2024 -0500
MAPREDUCE-7475. Fix non-idempotent unit tests (#6785)
Contributed by Kaiyao Ke
---
.../mapreduce/v2/app/webapp/TestAppController.java | 2 ++
.../java/org/apache/hadoop/mapred/TestMapTask.java | 18 ++++++++++++-----
.../hadoop/mapred/TestTaskProgressReporter.java | 6 ++++++
.../apache/hadoop/mapred/NotificationTestCase.java | 2 ++
.../hadoop/mapred/TestOldCombinerGrouping.java | 23 ++++++++++++++--------
.../hadoop/mapreduce/TestNewCombinerGrouping.java | 23 ++++++++++++++--------
6 files changed, 53 insertions(+), 21 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
index ba5c43012146..473681c3e424 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
@@ -319,6 +319,8 @@ public class TestAppController {
appController.attempts();
assertEquals(AttemptsPage.class, appController.getClazz());
+
+ appController.getProperty().remove(AMParams.ATTEMPT_STATE);
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java
index fef179994f09..771a5313ec32 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMapTask.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.Progress;
import org.junit.After;
+import org.junit.Before;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -47,14 +48,21 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
public class TestMapTask {
- private static File TEST_ROOT_DIR = new File(
+ private static File testRootDir = new File(
System.getProperty("test.build.data",
System.getProperty("java.io.tmpdir", "/tmp")),
TestMapTask.class.getName());
+ @Before
+ public void setup() throws Exception {
+ if(!testRootDir.exists()) {
+ testRootDir.mkdirs();
+ }
+ }
+
@After
public void cleanup() throws Exception {
- FileUtil.fullyDelete(TEST_ROOT_DIR);
+ FileUtil.fullyDelete(testRootDir);
}
@Rule
@@ -66,7 +74,7 @@ public class TestMapTask {
public void testShufflePermissions() throws Exception {
JobConf conf = new JobConf();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
- conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath());
+ conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath());
MapOutputFile mof = new MROutputFiles();
mof.setConf(conf);
TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1);
@@ -98,7 +106,7 @@ public class TestMapTask {
public void testSpillFilesCountLimitInvalidValue() throws Exception {
JobConf conf = new JobConf();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
- conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath());
+ conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath());
conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, -2);
MapOutputFile mof = new MROutputFiles();
mof.setConf(conf);
@@ -124,7 +132,7 @@ public class TestMapTask {
public void testSpillFilesCountBreach() throws Exception {
JobConf conf = new JobConf();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
- conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath());
+ conf.set(MRConfig.LOCAL_DIR, testRootDir.getAbsolutePath());
conf.setInt(MRJobConfig.SPILL_FILES_COUNT_LIMIT, 2);
MapOutputFile mof = new MROutputFiles();
mof.setConf(conf);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
index 52875b7aca70..93602935c9af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskProgressReporter.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
import org.apache.hadoop.util.ExitUtil;
import org.junit.After;
+import org.junit.Before;
import org.junit.Assert;
import org.junit.Test;
@@ -180,6 +181,11 @@ public class TestTaskProgressReporter {
}
}
+ @Before
+ public void setup() {
+ statusUpdateTimes = 0;
+ }
+
@After
public void cleanup() {
FileSystem.clearStatistics();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
index 3372c8f28b6f..8acd015ab098 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
@@ -158,6 +158,8 @@ public abstract class NotificationTestCase extends HadoopTestCase {
@After
public void tearDown() throws Exception {
stopHttpServer();
+ NotificationServlet.counter = 0;
+ NotificationServlet.failureCounter = 0;
super.tearDown();
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
index 046c2d37eed9..1f6395dfb789 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
@@ -18,11 +18,16 @@
package org.apache.hadoop.mapred;
+import org.junit.After;
import org.junit.Assert;
+
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Test;
import java.io.BufferedReader;
@@ -34,12 +39,9 @@ import java.io.PrintWriter;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
-import java.util.UUID;
public class TestOldCombinerGrouping {
- private static String TEST_ROOT_DIR = new File(System.getProperty(
- "test.build.data", "build/test/data"), UUID.randomUUID().toString())
- .getAbsolutePath();
+ private static File testRootDir = GenericTestUtils.getRandomizedTestDir();
public static class Map implements
Mapper<LongWritable, Text, Text, LongWritable> {
@@ -117,16 +119,21 @@ public class TestOldCombinerGrouping {
}
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(testRootDir);
+ }
+
@Test
public void testCombiner() throws Exception {
- if (!new File(TEST_ROOT_DIR).mkdirs()) {
- throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR);
+ if (!testRootDir.mkdirs()) {
+ throw new RuntimeException("Could not create test dir: " + testRootDir);
}
- File in = new File(TEST_ROOT_DIR, "input");
+ File in = new File(testRootDir, "input");
if (!in.mkdirs()) {
throw new RuntimeException("Could not create test dir: " + in);
}
- File out = new File(TEST_ROOT_DIR, "output");
+ File out = new File(testRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new File(in, "data.txt")));
pw.println("A|a,1");
pw.println("A|b,2");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
index c2054f1d4c1e..df9c6c5e9c19 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
@@ -18,7 +18,10 @@
package org.apache.hadoop.mapreduce;
+import org.junit.After;
import org.junit.Assert;
+
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
@@ -26,6 +29,8 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
import java.io.BufferedReader;
@@ -36,12 +41,9 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashSet;
import java.util.Set;
-import java.util.UUID;
public class TestNewCombinerGrouping {
- private static String TEST_ROOT_DIR = new File(System.getProperty(
- "test.build.data", "build/test/data"), UUID.randomUUID().toString())
- .getAbsolutePath();
+ private static File testRootDir = GenericTestUtils.getRandomizedTestDir();
public static class Map extends
Mapper<LongWritable, Text, Text, LongWritable> {
@@ -103,16 +105,21 @@ public class TestNewCombinerGrouping {
}
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(testRootDir);
+ }
+
@Test
public void testCombiner() throws Exception {
- if (!new File(TEST_ROOT_DIR).mkdirs()) {
- throw new RuntimeException("Could not create test dir: " + TEST_ROOT_DIR);
+ if (!testRootDir.mkdirs()) {
+ throw new RuntimeException("Could not create test dir: " + testRootDir);
}
- File in = new File(TEST_ROOT_DIR, "input");
+ File in = new File(testRootDir, "input");
if (!in.mkdirs()) {
throw new RuntimeException("Could not create test dir: " + in);
}
- File out = new File(TEST_ROOT_DIR, "output");
+ File out = new File(testRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new File(in, "data.txt")));
pw.println("A|a,1");
pw.println("A|b,2");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org