You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2016/03/29 04:33:09 UTC
hadoop git commit: MAPREDUCE-6662. Clear ASF Warnings on test data
files. Contributed by Vinayakumar B
Repository: hadoop
Updated Branches:
refs/heads/trunk 948b75807 -> 0ef8bbfd8
MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ef8bbfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ef8bbfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ef8bbfd
Branch: refs/heads/trunk
Commit: 0ef8bbfd8791899c7cfed3dd9c1670182fd87575
Parents: 948b758
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Mar 29 10:32:21 2016 +0800
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Mar 29 10:32:21 2016 +0800
----------------------------------------------------------------------
.../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +++--
.../apache/hadoop/mapred/TestFieldSelection.java | 5 +++--
.../mapred/TestMRTimelineEventHandling.java | 19 ++++++++++++-------
.../hadoop/mapred/TestMiniMRChildTask.java | 16 ++++++++++------
.../hadoop/mapred/TestOldCombinerGrouping.java | 5 +++--
.../mapred/lib/aggregate/TestAggregates.java | 7 ++++---
.../mapreduce/TestNewCombinerGrouping.java | 5 +++--
.../lib/aggregate/TestMapReduceAggregates.java | 5 +++--
.../lib/input/TestCombineFileInputFormat.java | 4 +++-
9 files changed, 44 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 18e3e6a..aa2e979 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -71,8 +71,9 @@ public class TestHistoryFileManager {
Configuration conf = new HdfsConfiguration();
Configuration conf2 = new HdfsConfiguration();
dfsCluster = new MiniDFSCluster.Builder(conf).build();
- conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,
- conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2");
+ conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get(
+ MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory())
+ + "_2");
dfsCluster2 = new MiniDFSCluster.Builder(conf2).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
index 239c239..29333b7b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
@@ -44,8 +44,9 @@ private static NumberFormat idFormat = NumberFormat.getInstance();
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
- Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
- Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
+ String baseDir = System.getProperty("test.build.data", "build/test/data");
+ Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test");
+ Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index a82007f..40ed9ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -97,9 +97,10 @@ public class TestMRTimelineEventHandling {
+ cluster.getApplicationHistoryServer().getPort());
TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore();
-
- Path inDir = new Path("input");
- Path outDir = new Path("output");
+ String localPathRoot = System.getProperty("test.build.data",
+ "build/test/data");
+ Path inDir = new Path(localPathRoot, "input");
+ Path outDir = new Path(localPathRoot, "output");
RunningJob job =
UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED,
@@ -155,8 +156,10 @@ public class TestMRTimelineEventHandling {
TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore();
- Path inDir = new Path("input");
- Path outDir = new Path("output");
+ String localPathRoot = System.getProperty("test.build.data",
+ "build/test/data");
+ Path inDir = new Path(localPathRoot, "input");
+ Path outDir = new Path(localPathRoot, "output");
RunningJob job =
UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED,
@@ -195,8 +198,10 @@ public class TestMRTimelineEventHandling {
TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore();
- Path inDir = new Path("input");
- Path outDir = new Path("output");
+ String localPathRoot = System.getProperty("test.build.data",
+ "build/test/data");
+ Path inDir = new Path(localPathRoot, "input");
+ Path outDir = new Path(localPathRoot, "output");
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
RunningJob job =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 74be8a7..cbeeccf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -436,8 +436,10 @@ public class TestMiniMRChildTask {
JobConf conf = new JobConf(mr.getConfig());
// initialize input, output directories
- Path inDir = new Path("input");
- Path outDir = new Path("output");
+ Path rootDir = new Path(System.getProperty("test.build.data",
+ "build/test/data"));
+ Path inDir = new Path(rootDir, "input");
+ Path outDir = new Path(rootDir, "output");
String input = "The input";
// set config to use the ExecutionEnvCheckMapClass map class
@@ -462,9 +464,10 @@ public class TestMiniMRChildTask {
public void testTaskEnv(){
try {
JobConf conf = new JobConf(mr.getConfig());
+ String baseDir = System.getProperty("test.build.data", "build/test/data");
// initialize input, output directories
- Path inDir = new Path("testing/wc/input1");
- Path outDir = new Path("testing/wc/output1");
+ Path inDir = new Path(baseDir + "/testing/wc/input1");
+ Path outDir = new Path(baseDir + "/testing/wc/output1");
FileSystem outFs = outDir.getFileSystem(conf);
runTestTaskEnv(conf, inDir, outDir, false);
outFs.delete(outDir, true);
@@ -485,9 +488,10 @@ public class TestMiniMRChildTask {
public void testTaskOldEnv(){
try {
JobConf conf = new JobConf(mr.getConfig());
+ String baseDir = System.getProperty("test.build.data", "build/test/data");
// initialize input, output directories
- Path inDir = new Path("testing/wc/input1");
- Path outDir = new Path("testing/wc/output1");
+ Path inDir = new Path(baseDir + "/testing/wc/input1");
+ Path outDir = new Path(baseDir + "/testing/wc/output1");
FileSystem outFs = outDir.getFileSystem(conf);
runTestTaskEnv(conf, inDir, outDir, true);
outFs.delete(outDir, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
index 80a6702..046c2d3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
@@ -37,8 +37,9 @@ import java.util.Set;
import java.util.UUID;
public class TestOldCombinerGrouping {
- private static String TEST_ROOT_DIR =
- new File("build", UUID.randomUUID().toString()).getAbsolutePath();
+ private static String TEST_ROOT_DIR = new File(System.getProperty(
+ "test.build.data", "build/test/data"), UUID.randomUUID().toString())
+ .getAbsolutePath();
public static class Map implements
Mapper<LongWritable, Text, Text, LongWritable> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
index 6da96ce..e2fdd42 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
@@ -46,8 +46,9 @@ public class TestAggregates extends TestCase {
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
- Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
- Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
+ String baseDir = System.getProperty("test.build.data", "build/test/data");
+ Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
+ Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
@@ -115,7 +116,7 @@ public class TestAggregates extends TestCase {
outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata);
- //fs.delete(OUTPUT_DIR);
+ fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
index ab9ddbb..c2054f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
@@ -39,8 +39,9 @@ import java.util.Set;
import java.util.UUID;
public class TestNewCombinerGrouping {
- private static String TEST_ROOT_DIR =
- new File("build", UUID.randomUUID().toString()).getAbsolutePath();
+ private static String TEST_ROOT_DIR = new File(System.getProperty(
+ "test.build.data", "build/test/data"), UUID.randomUUID().toString())
+ .getAbsolutePath();
public static class Map extends
Mapper<LongWritable, Text, Text, LongWritable> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
index f24dffe..788ad41 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
@@ -51,8 +51,9 @@ public class TestMapReduceAggregates extends TestCase {
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20;
- Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
- Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
+ String baseDir = System.getProperty("test.build.data", "build/test/data");
+ Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
+ Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ef8bbfd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
index b49f2d8..1fca5c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
@@ -1772,7 +1772,9 @@ public class TestCombineFileInputFormat {
// default fs path
assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString());
// add a local file
- Path localPath = new Path("testFile1");
+ String localPathRoot = System.getProperty("test.build.data",
+ "build/test/data");
+ Path localPath = new Path(localPathRoot, "testFile1");
FileSystem lfs = FileSystem.getLocal(conf);
FSDataOutputStream dos = lfs.create(localPath);
dos.writeChars("Local file for CFIF");