You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/08/27 17:49:43 UTC
[1/7] git commit: MAPREDUCE-5885. build/test/test.mapred.spill causes
release audit warnings. Contributed by Chen He
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 6b015989f -> 7ac5b6e19
MAPREDUCE-5885. build/test/test.mapred.spill causes release audit warnings. Contributed by Chen He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ac5b6e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ac5b6e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ac5b6e1
Branch: refs/heads/HDFS-6581
Commit: 7ac5b6e19b19047283e944e2af75c6cc79c96c1b
Parents: 09b6482
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 27 15:14:54 2014 +0000
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../apache/hadoop/mapred/TestComparators.java | 61 +++++++++++++------
.../apache/hadoop/mapred/TestMapOutputType.java | 62 ++++++++++++--------
.../org/apache/hadoop/mapred/TestMapRed.java | 28 ++++++---
.../mapred/lib/TestKeyFieldBasedComparator.java | 25 +++++++-
.../apache/hadoop/mapreduce/TestMapReduce.java | 30 +++++++---
6 files changed, 150 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a6d2981..de0767d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -258,6 +258,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir
creation on Windows. (zjshen)
+ MAPREDUCE-5885. build/test/test.mapred.spill causes release audit warnings
+ (Chen He via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java
index 1cef5cb..f83dbe2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestComparators.java
@@ -17,13 +17,30 @@
*/
package org.apache.hadoop.mapred;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.MRConfig;
-import junit.framework.TestCase;
-import java.io.*;
-import java.util.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
/**
* Two different types of comparators can be used in MapReduce. One is used
@@ -37,8 +54,11 @@ import java.util.*;
* 2. Test the common use case where values are grouped by keys but values
* within each key are grouped by a secondary key (a timestamp, for example).
*/
-public class TestComparators extends TestCase
-{
+public class TestComparators {
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")), "TestComparators-mapred");
+
JobConf conf = new JobConf(TestMapOutputType.class);
JobClient jc;
static Random rng = new Random();
@@ -292,9 +312,9 @@ public class TestComparators extends TestCase
}
}
-
+ @Before
public void configure() throws Exception {
- Path testdir = new Path("build/test/test.mapred.spill");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.get(conf);
@@ -334,14 +354,18 @@ public class TestComparators extends TestCase
jc = new JobClient(conf);
}
-
+
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
/**
* Test the default comparator for Map/Reduce.
* Use the identity mapper and see if the keys are sorted at the end
* @throws Exception
*/
- public void testDefaultMRComparator() throws Exception {
- configure();
+ @Test
+ public void testDefaultMRComparator() throws Exception {
conf.setMapperClass(IdentityMapper.class);
conf.setReducerClass(AscendingKeysReducer.class);
@@ -361,8 +385,8 @@ public class TestComparators extends TestCase
* comparator. Keys should be sorted in reverse order in the reducer.
* @throws Exception
*/
- public void testUserMRComparator() throws Exception {
- configure();
+ @Test
+ public void testUserMRComparator() throws Exception {
conf.setMapperClass(IdentityMapper.class);
conf.setReducerClass(DescendingKeysReducer.class);
conf.setOutputKeyComparatorClass(DecreasingIntComparator.class);
@@ -384,8 +408,8 @@ public class TestComparators extends TestCase
* values for a key should be sorted by the 'timestamp'.
* @throws Exception
*/
- public void testUserValueGroupingComparator() throws Exception {
- configure();
+ @Test
+ public void testUserValueGroupingComparator() throws Exception {
conf.setMapperClass(RandomGenMapper.class);
conf.setReducerClass(AscendingGroupReducer.class);
conf.setOutputValueGroupingComparator(CompositeIntGroupFn.class);
@@ -409,8 +433,8 @@ public class TestComparators extends TestCase
* order. This lets us make sure that the right comparators are used.
* @throws Exception
*/
- public void testAllUserComparators() throws Exception {
- configure();
+ @Test
+ public void testAllUserComparators() throws Exception {
conf.setMapperClass(RandomGenMapper.class);
// use a decreasing comparator so keys are sorted in reverse order
conf.setOutputKeyComparatorClass(DecreasingIntComparator.class);
@@ -430,6 +454,7 @@ public class TestComparators extends TestCase
* Test a user comparator that relies on deserializing both arguments
* for each compare.
*/
+ @Test
public void testBakedUserComparator() throws Exception {
MyWritable a = new MyWritable(8, 8);
MyWritable b = new MyWritable(7, 9);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java
index d11d7bc..e3860fd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapOutputType.java
@@ -17,21 +17,36 @@
*/
package org.apache.hadoop.mapred;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.lib.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.MRConfig;
-import junit.framework.TestCase;
-import java.io.*;
-import java.util.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
/**
* TestMapOutputType checks whether the Map task handles type mismatch
* between mapper output and the type specified in
* JobConf.MapOutputKeyType and JobConf.MapOutputValueType.
*/
-public class TestMapOutputType extends TestCase
-{
+public class TestMapOutputType {
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")), "TestMapOutputType-mapred");
JobConf conf = new JobConf(TestMapOutputType.class);
JobClient jc;
/**
@@ -75,9 +90,9 @@ public class TestMapOutputType extends TestCase
}
}
-
+ @Before
public void configure() throws Exception {
- Path testdir = new Path("build/test/test.mapred.spill");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.get(conf);
@@ -101,17 +116,21 @@ public class TestMapOutputType extends TestCase
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
Path inFile = new Path(inDir, "part0");
- SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile,
+ SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile,
Text.class, Text.class);
writer.append(new Text("rec: 1"), new Text("Hello"));
writer.close();
jc = new JobClient(conf);
}
-
+
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
+
+ @Test
public void testKeyMismatch() throws Exception {
- configure();
-
// Set bad MapOutputKeyClass and MapOutputValueClass
conf.setMapOutputKeyClass(IntWritable.class);
conf.setMapOutputValueClass(IntWritable.class);
@@ -125,11 +144,9 @@ public class TestMapOutputType extends TestCase
fail("Oops! The job was supposed to break due to an exception");
}
}
-
+
+ @Test
public void testValueMismatch() throws Exception {
- configure();
-
- // Set good MapOutputKeyClass, bad MapOutputValueClass
conf.setMapOutputKeyClass(Text.class);
conf.setMapOutputValueClass(IntWritable.class);
@@ -142,11 +159,10 @@ public class TestMapOutputType extends TestCase
fail("Oops! The job was supposed to break due to an exception");
}
}
-
- public void testNoMismatch() throws Exception{
- configure();
-
- // Set good MapOutputKeyClass and MapOutputValueClass
+
+ @Test
+ public void testNoMismatch() throws Exception{
+ // Set good MapOutputKeyClass and MapOutputValueClass
conf.setMapOutputKeyClass(Text.class);
conf.setMapOutputValueClass(Text.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
index 3f7a6f7..02a083b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
@@ -24,7 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
-import java.util.Collections;
+import java.io.File;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
@@ -46,11 +47,11 @@ import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
/**********************************************************
* MapredLoadTest generates a bunch of work that exercises
@@ -110,6 +111,10 @@ public class TestMapRed extends Configured implements Tool {
* of numbers in random order, but where each number appears
* as many times as we were instructed.
*/
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")), "TestMapRed-mapred");
+
static class RandomGenMapper
implements Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
@@ -248,6 +253,11 @@ public class TestMapRed extends Configured implements Tool {
private static int counts = 100;
private static Random r = new Random();
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
+
/**
public TestMapRed(int range, int counts, Configuration conf) throws IOException {
this.range = range;
@@ -372,7 +382,7 @@ public class TestMapRed extends Configured implements Tool {
boolean includeCombine
) throws Exception {
JobConf conf = new JobConf(TestMapRed.class);
- Path testdir = new Path("build/test/test.mapred.compress");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.get(conf);
@@ -440,7 +450,7 @@ public class TestMapRed extends Configured implements Tool {
//
// Generate distribution of ints. This is the answer key.
//
- JobConf conf = null;
+ JobConf conf;
//Check to get configuration and check if it is configured thro' Configured
//interface. This would happen when running testcase thro' command line.
if(getConf() == null) {
@@ -465,7 +475,7 @@ public class TestMapRed extends Configured implements Tool {
// Write the answer key to a file.
//
FileSystem fs = FileSystem.get(conf);
- Path testdir = new Path("mapred.loadtest");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath(), "mapred.loadtest");
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());
}
@@ -635,8 +645,8 @@ public class TestMapRed extends Configured implements Tool {
in.close();
}
int originalTotal = 0;
- for (int i = 0; i < dist.length; i++) {
- originalTotal += dist[i];
+ for (int aDist : dist) {
+ originalTotal += aDist;
}
System.out.println("Original sum: " + originalTotal);
System.out.println("Recomputed sum: " + totalseen);
@@ -727,7 +737,7 @@ public class TestMapRed extends Configured implements Tool {
public void runJob(int items) {
try {
JobConf conf = new JobConf(TestMapRed.class);
- Path testdir = new Path("build/test/test.mapred.spill");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.get(conf);
@@ -777,7 +787,7 @@ public class TestMapRed extends Configured implements Tool {
System.err.println("Usage: TestMapRed <range> <counts>");
System.err.println();
System.err.println("Note: a good test will have a " +
- "<counts> value that is substantially larger than the <range>");
+ "<counts> value that is substantially larger than the <range>");
return -1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
index 0bee2b5..34a4d2c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.mapred.lib;
-import java.io.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -35,9 +34,23 @@ import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.Utils;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
public class TestKeyFieldBasedComparator extends HadoopTestCase {
+
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")),
+ "TestKeyFieldBasedComparator-lib");
JobConf conf;
JobConf localConf;
@@ -50,8 +63,9 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase {
localConf = createJobConf();
localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " ");
}
+
public void configure(String keySpec, int expect) throws Exception {
- Path testdir = new Path("build/test/test.mapred.spill");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = getFileSystem();
@@ -116,6 +130,13 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase {
reader.close();
}
}
+
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
+
+ @Test
public void testBasicUnixComparator() throws Exception {
configure("-k1,1n", 1);
configure("-k2,2n", 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac5b6e1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
index 01e1283..48ad47a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduce.java
@@ -23,14 +23,14 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
+import java.io.File;
import java.util.Iterator;
import java.util.Random;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
@@ -41,6 +41,10 @@ import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.junit.After;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
/**********************************************************
* MapredLoadTest generates a bunch of work that exercises
@@ -75,8 +79,10 @@ import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
* 7) A mapred job integrates all the count files into a single one.
*
**********************************************************/
-public class TestMapReduce extends TestCase {
-
+public class TestMapReduce {
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")), "TestMapReduce-mapreduce");
private static FileSystem fs;
static {
@@ -215,6 +221,12 @@ public class TestMapReduce extends TestCase {
private static int counts = 100;
private static Random r = new Random();
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
+
+ @Test
public void testMapred() throws Exception {
launch();
}
@@ -239,7 +251,7 @@ public class TestMapReduce extends TestCase {
//
// Write the answer key to a file.
//
- Path testdir = new Path("mapred.loadtest");
+ Path testdir = new Path(TEST_DIR.getAbsolutePath());
if (!fs.mkdirs(testdir)) {
throw new IOException("Mkdirs failed to create " + testdir.toString());
}
@@ -488,13 +500,17 @@ public class TestMapReduce extends TestCase {
System.err.println("Usage: TestMapReduce <range> <counts>");
System.err.println();
System.err.println("Note: a good test will have a <counts> value" +
- " that is substantially larger than the <range>");
+ " that is substantially larger than the <range>");
return;
}
int i = 0;
range = Integer.parseInt(argv[i++]);
counts = Integer.parseInt(argv[i++]);
- launch();
+ try {
+ launch();
+ } finally {
+ FileUtil.fullyDelete(TEST_DIR);
+ }
}
}
[4/7] git commit: Add a section for 2.5.1 in CHANGES.txt
Posted by ar...@apache.org.
Add a section for 2.5.1 in CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcb16deb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcb16deb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcb16deb
Branch: refs/heads/HDFS-6581
Commit: fcb16debb20b2647666033430cde4e810d9cf93a
Parents: 6b01598
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Aug 27 00:55:19 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 12 ++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++
hadoop-mapreduce-project/CHANGES.txt | 12 ++++++++++++
hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++
4 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcb16deb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0291c75..9242ca4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -695,6 +695,18 @@ Release 2.6.0 - UNRELEASED
HADOOP-10989. Work around buggy getgrouplist() implementations on Linux that
return 0 on failure. (cnauroth)
+Release 2.5.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcb16deb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 31ed15c..4e60c46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -638,6 +638,18 @@ Release 2.6.0 - UNRELEASED
HDFS-4852. libhdfs documentation is out of date. (cnauroth)
+Release 2.5.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcb16deb/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index ddf21ed..4cd71c0 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -258,6 +258,18 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6044. Fully qualified intermediate done dir path breaks per-user dir
creation on Windows. (zjshen)
+Release 2.5.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcb16deb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5b61b41..916816e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -237,6 +237,18 @@ Release 2.6.0 - UNRELEASED
YARN-2434. RM should not recover containers from previously failed attempt
when AM restart is not enabled (Jian He via jlowe)
+Release 2.5.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
[7/7] git commit: YARN-2035. FileSystemApplicationHistoryStore should
not make working dir when it already exists. Contributed by Jonathan Eagles.
Posted by ar...@apache.org.
YARN-2035. FileSystemApplicationHistoryStore should not make working dir when it already exists. Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9d319c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9d319c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9d319c0
Branch: refs/heads/HDFS-6581
Commit: c9d319c05ff2044928192c9b65e7f7f758b5a5f6
Parents: 25df42d
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Aug 27 02:01:00 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../FileSystemApplicationHistoryStore.java | 14 ++++-
.../TestFileSystemApplicationHistoryStore.java | 62 +++++++++++++++++++-
3 files changed, 75 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9d319c0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eefa547..36d304c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -240,6 +240,9 @@ Release 2.6.0 - UNRELEASED
YARN-2434. RM should not recover containers from previously failed attempt
when AM restart is not enabled (Jian He via jlowe)
+ YARN-2035. FileSystemApplicationHistoryStore should not make working dir
+ when it already exists. (Jonathan Eagles via zjshen)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9d319c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
index a5725eb..a2d9140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
@@ -110,15 +110,23 @@ public class FileSystemApplicationHistoryStore extends AbstractService
super(FileSystemApplicationHistoryStore.class.getName());
}
+ protected FileSystem getFileSystem(Path path, Configuration conf) throws Exception {
+ return path.getFileSystem(conf);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
Path fsWorkingPath =
new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI));
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
try {
- fs = fsWorkingPath.getFileSystem(conf);
- fs.mkdirs(rootDirPath);
- fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ fs = getFileSystem(fsWorkingPath, conf);
+
+ if (!fs.isDirectory(rootDirPath)) {
+ fs.mkdirs(rootDirPath);
+ fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ }
+
} catch (IOException e) {
LOG.error("Error when initializing FileSystemHistoryStorage", e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9d319c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
index d31018c..552a5e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
@@ -20,9 +20,17 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import org.junit.Assert;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -53,6 +61,11 @@ public class TestFileSystemApplicationHistoryStore extends
@Before
public void setup() throws Exception {
fs = new RawLocalFileSystem();
+ initStore(fs);
+ }
+
+ private void initStore(final FileSystem fs) throws IOException,
+ URISyntaxException {
Configuration conf = new Configuration();
fs.initialize(new URI("/"), conf);
fsWorkingPath =
@@ -61,7 +74,12 @@ public class TestFileSystemApplicationHistoryStore extends
fs.delete(fsWorkingPath, true);
conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
fsWorkingPath.toString());
- store = new FileSystemApplicationHistoryStore();
+ store = new FileSystemApplicationHistoryStore() {
+ @Override
+ protected FileSystem getFileSystem(Path path, Configuration conf) {
+ return fs;
+ }
+ };
store.init(conf);
store.start();
}
@@ -243,4 +261,46 @@ public class TestFileSystemApplicationHistoryStore extends
testWriteHistoryData(3, false, true);
testReadHistoryData(3, false, true);
}
+
+ @Test
+ public void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
+ LOG.info("Starting testInitExistingWorkingDirectoryInSafeMode");
+ tearDown();
+
+ // Setup file system to inject startup conditions
+ FileSystem fs = spy(new RawLocalFileSystem());
+ doReturn(true).when(fs).isDirectory(any(Path.class));
+
+ try {
+ initStore(fs);
+ } catch (Exception e) {
+ Assert.fail("Exception should not be thrown: " + e);
+ }
+
+ // Make sure that directory creation was not attempted
+ verify(fs, times(1)).isDirectory(any(Path.class));
+ verify(fs, times(0)).mkdirs(any(Path.class));
+ }
+
+ @Test
+ public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception {
+ LOG.info("Starting testInitNonExistingWorkingDirectoryInSafeMode");
+ tearDown();
+
+ // Setup file system to inject startup conditions
+ FileSystem fs = spy(new RawLocalFileSystem());
+ doReturn(false).when(fs).isDirectory(any(Path.class));
+ doThrow(new IOException()).when(fs).mkdirs(any(Path.class));
+
+ try {
+ initStore(fs);
+ Assert.fail("Exception should have been thrown");
+ } catch (Exception e) {
+ // Expected failure
+ }
+
+ // Make sure that directory creation was attempted
+ verify(fs, times(1)).isDirectory(any(Path.class));
+ verify(fs, times(1)).mkdirs(any(Path.class));
+ }
}
[3/7] git commit: Fix CHANGES.txt entry for MAPREDUCE-6033.
Posted by ar...@apache.org.
Fix CHANGES.txt entry for MAPREDUCE-6033.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46f63818
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46f63818
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46f63818
Branch: refs/heads/HDFS-6581
Commit: 46f638183621b5cc11cdc9556a85043dc29a3984
Parents: fcb16de
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Aug 27 01:16:52 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f63818/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4cd71c0..a6d2981 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -270,6 +270,9 @@ Release 2.5.1 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6033. Updated access check for displaying job information
+ (Yu Gao via Eric Yang)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
@@ -351,9 +354,6 @@ Release 2.5.0 - 2014-08-11
BUG FIXES
- MAPREDUCE-6033. Updated access check for displaying job information
- (Yu Gao via Eric Yang)
-
MAPREDUCE-5759. Remove unnecessary conf load in Limits (Sandy Ryza)
MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing.
[6/7] git commit: HADOOP-10996. Stop violence in the *_HOME (aw)
Posted by ar...@apache.org.
HADOOP-10996. Stop violence in the *_HOME (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09b64821
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09b64821
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09b64821
Branch: refs/heads/HDFS-6581
Commit: 09b64821d8f20015ff008b0f920916ef2c895a0c
Parents: 2d617e9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Aug 27 07:00:31 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop-common/src/main/bin/hadoop-config.sh | 10 ++++++++--
.../src/main/bin/hadoop-functions.sh | 12 +++++-------
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 15 ++++++++-------
hadoop-mapreduce-project/bin/mapred-config.sh | 20 +++++++++++---------
.../hadoop-yarn/bin/yarn-config.sh | 16 ++++++++--------
6 files changed, 42 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2270df3..45e38d3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -321,6 +321,8 @@ Trunk (Unreleased)
HADOOP-11002. shell escapes are incompatible with previous releases (aw)
+ HADOOP-10996. Stop violence in the *_HOME (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index b2fc4d3..0cf8bcf 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -53,7 +53,10 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
fi
# get our functions defined for usage later
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh" ]]; then
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
else
echo "ERROR: Unable to exec ${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh." 1>&2
@@ -61,7 +64,10 @@ else
fi
# allow overrides of the above and pre-defines of the below
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh" ]]; then
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh" ]]; then
. "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index ab61b84..800e024 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -68,17 +68,18 @@ function hadoop_find_confdir
{
# NOTE: This function is not user replaceable.
+ local conf_dir
# Look for the basic hadoop configuration area.
#
#
# An attempt at compatibility with some Hadoop 1.x
# installs.
if [[ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]]; then
- DEFAULT_CONF_DIR="conf"
+ conf_dir="conf"
else
- DEFAULT_CONF_DIR="etc/hadoop"
+ conf_dir="etc/hadoop"
fi
- export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_PREFIX}/${DEFAULT_CONF_DIR}}"
+ export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_PREFIX}/${conf_dir}}"
}
function hadoop_exec_hadoopenv
@@ -573,10 +574,7 @@ function hadoop_finalize_hadoop_opts
function hadoop_finalize_classpath
{
-
- # we want the HADOOP_CONF_DIR at the end
- # according to oom, it gives a 2% perf boost
- hadoop_add_classpath "${HADOOP_CONF_DIR}" after
+ hadoop_add_classpath "${HADOOP_CONF_DIR}" before
# user classpath gets added at the last minute. this allows
# override of CONF dirs and more
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
index fb460d9..6824028 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -20,7 +20,7 @@
function hadoop_subproject_init
{
- if [ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]; then
+ if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then
. "${HADOOP_CONF_DIR}/hdfs-env.sh"
fi
@@ -49,7 +49,7 @@ function hadoop_subproject_init
HADOOP_ROOT_LOGGER=${HADOOP_HDFS_ROOT_LOGGER:-$HADOOP_ROOT_LOGGER}
HADOOP_HDFS_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
- HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME_DIR}"
+ HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_PREFIX}"
HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}"
HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}"
@@ -71,12 +71,13 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P)
fi
-if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
-elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
- . "${HADOOP_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
else
echo "ERROR: Hadoop common not found." 2>&1
exit 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-mapreduce-project/bin/mapred-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
index c24d350..c2681ac 100644
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -20,7 +20,7 @@
function hadoop_subproject_init
{
- if [ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then
+ if [[ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]]; then
. "${HADOOP_CONF_DIR}/mapred-env.sh"
fi
@@ -49,7 +49,7 @@ function hadoop_subproject_init
HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}"
HADOOP_MAPRED_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
- HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_HOME_DIR}"
+ HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}"
HADOOP_IDENT_STRING="${HADOOP_MAPRED_IDENT_STRING:-$HADOOP_IDENT_STRING}"
HADOOP_MAPRED_IDENT_STRING="${HADOOP_IDENT_STRING}"
@@ -60,13 +60,15 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_mc_this}")" >/dev/null && pwd -P)
fi
-if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
-elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
else
- echo "Hadoop common not found."
- exit
+ echo "ERROR: Hadoop common not found." 2>&1
+ exit 1
fi
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b64821/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
index 34d2d2d..d83e998 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
@@ -80,14 +80,14 @@ if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_yc_this}")" >/dev/null && pwd -P)
fi
-if [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-elif [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
. "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
-elif [[ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]]; then
- . "${HADOOP_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]; then
+ . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
else
- echo "Hadoop common not found."
- exit
+ echo "ERROR: Hadoop common not found." 2>&1
+ exit 1
fi
-
[2/7] git commit: HADOOP-11002. shell escapes are incompatible with
previous releases (aw)
Posted by ar...@apache.org.
HADOOP-11002. shell escapes are incompatible with previous releases (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d617e9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d617e9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d617e9e
Branch: refs/heads/HDFS-6581
Commit: 2d617e9ee75ea49fffceda14cf51ed7abd23975d
Parents: c9d319c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 26 14:40:46 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/main/bin/hadoop-functions.sh | 31 ++++++--------------
2 files changed, 11 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d617e9e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9242ca4..2270df3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -319,6 +319,8 @@ Trunk (Unreleased)
HADOOP-10925. Compilation fails in native link0 function on Windows.
(cnauroth)
+ HADOOP-11002. shell escapes are incompatible with previous releases (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d617e9e/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index f2437fa..ab61b84 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -59,8 +59,7 @@ function hadoop_bootstrap_init
TOOL_PATH=${TOOL_PATH:-${HADOOP_PREFIX}/share/hadoop/tools/lib/*}
export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
-
-
+
# defaults
export HADOOP_OPTS=${HADOOP_OPTS:-"-Djava.net.preferIPv4Stack=true"}
}
@@ -94,7 +93,6 @@ function hadoop_exec_hadoopenv
fi
}
-
function hadoop_basic_init
{
# Some of these are also set in hadoop-env.sh.
@@ -446,7 +444,6 @@ function hadoop_add_to_classpath_mapred
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
}
-
function hadoop_add_to_classpath_userpath
{
# Add the user-specified HADOOP_CLASSPATH to the
@@ -551,7 +548,6 @@ function hadoop_java_setup
fi
}
-
function hadoop_finalize_libpaths
{
if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
@@ -564,17 +560,14 @@ function hadoop_finalize_libpaths
#
# fill in any last minute options that might not have been defined yet
#
-# Note that we are replacing ' ' with '\ ' so that directories with
-# spaces work correctly when run exec blah
-#
function hadoop_finalize_hadoop_opts
{
- hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR/ /\ }"
- hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE/ /\ }"
- hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX/ /\ }"
- hadoop_add_param HADOOP_OPTS hadoop.id.str "-Dhadoop.id.str=${HADOOP_IDENT_STRING/ /\ }"
+ hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR}"
+ hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE}"
+ hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}"
+ hadoop_add_param HADOOP_OPTS hadoop.id.str "-Dhadoop.id.str=${HADOOP_IDENT_STRING}"
hadoop_add_param HADOOP_OPTS hadoop.root.logger "-Dhadoop.root.logger=${HADOOP_ROOT_LOGGER}"
- hadoop_add_param HADOOP_OPTS hadoop.policy.file "-Dhadoop.policy.file=${HADOOP_POLICYFILE/ /\ }"
+ hadoop_add_param HADOOP_OPTS hadoop.policy.file "-Dhadoop.policy.file=${HADOOP_POLICYFILE}"
hadoop_add_param HADOOP_OPTS hadoop.security.logger "-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER}"
}
@@ -724,10 +717,8 @@ function hadoop_java_exec
local command=$1
local class=$2
shift 2
- # we eval this so that paths with spaces work
#shellcheck disable=SC2086
- eval exec "$JAVA" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
-
+ exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
}
function hadoop_start_daemon
@@ -739,7 +730,7 @@ function hadoop_start_daemon
local class=$2
shift 2
#shellcheck disable=SC2086
- eval exec "$JAVA" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
+ exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
}
function hadoop_start_daemon_wrapper
@@ -802,9 +793,7 @@ function hadoop_start_secure_daemon
# where to send stderr. same thing, except &2 = stderr
local daemonerrfile=$5
shift 5
-
-
-
+
hadoop_rotate_log "${daemonoutfile}"
hadoop_rotate_log "${daemonerrfile}"
@@ -925,7 +914,6 @@ function hadoop_stop_daemon
fi
}
-
function hadoop_stop_secure_daemon
{
local command=$1
@@ -984,7 +972,6 @@ function hadoop_daemon_handler
esac
}
-
function hadoop_secure_daemon_handler
{
local daemonmode=$1
[5/7] git commit: YARN-1326. RM should log using RMStore at startup
time. (Tsuyoshi Ozawa via kasha)
Posted by ar...@apache.org.
YARN-1326. RM should log using RMStore at startup time. (Tsuyoshi Ozawa via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25df42d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25df42d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25df42d6
Branch: refs/heads/HDFS-6581
Commit: 25df42d6e7d10cd48f20ac647a6bf417a8185c6b
Parents: 46f6381
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Aug 27 01:43:58 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Wed Aug 27 08:49:18 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/recovery/RMStateStoreFactory.java | 13 ++++++++-----
.../yarn/server/resourcemanager/webapp/AboutBlock.java | 1 +
.../server/resourcemanager/webapp/dao/ClusterInfo.java | 8 ++++++++
.../resourcemanager/webapp/TestRMWebServices.java | 2 +-
5 files changed, 21 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25df42d6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 916816e..eefa547 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -154,6 +154,9 @@ Release 2.6.0 - UNRELEASED
YARN-2389. Added functionality for schedulers to kill all applications in a
queue. (Subramaniam Venkatraman Krishnan via jianhe)
+ YARN-1326. RM should log using RMStore at startup time.
+ (Tsuyoshi Ozawa via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25df42d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreFactory.java
index f9e2869..c09ddb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreFactory.java
@@ -17,17 +17,20 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
public class RMStateStoreFactory {
+ private static final Log LOG = LogFactory.getLog(RMStateStoreFactory.class);
public static RMStateStore getStore(Configuration conf) {
- RMStateStore store = ReflectionUtils.newInstance(
- conf.getClass(YarnConfiguration.RM_STORE,
- MemoryRMStateStore.class, RMStateStore.class),
- conf);
- return store;
+ Class<? extends RMStateStore> storeClass =
+ conf.getClass(YarnConfiguration.RM_STORE,
+ MemoryRMStateStore.class, RMStateStore.class);
+ LOG.info("Using RMStateStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25df42d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
index 91b5cc1..ea5c48a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
@@ -44,6 +44,7 @@ public class AboutBlock extends HtmlBlock {
_("Cluster ID:", cinfo.getClusterId()).
_("ResourceManager state:", cinfo.getState()).
_("ResourceManager HA state:", cinfo.getHAState()).
+ _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
_("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
_("ResourceManager version:", cinfo.getRMBuildVersion() +
" on " + cinfo.getRMVersionBuiltOn()).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25df42d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
index c96d73e..b529f21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@XmlRootElement
@@ -35,6 +36,7 @@ public class ClusterInfo {
protected long startedOn;
protected STATE state;
protected HAServiceProtocol.HAServiceState haState;
+ protected String rmStateStoreName;
protected String resourceManagerVersion;
protected String resourceManagerBuildVersion;
protected String resourceManagerVersionBuiltOn;
@@ -51,6 +53,8 @@ public class ClusterInfo {
this.id = ts;
this.state = rm.getServiceState();
this.haState = rm.getRMContext().getHAServiceState();
+ this.rmStateStoreName = rm.getRMContext().getStateStore().getClass()
+ .getName();
this.startedOn = ts;
this.resourceManagerVersion = YarnVersionInfo.getVersion();
this.resourceManagerBuildVersion = YarnVersionInfo.getBuildVersion();
@@ -68,6 +72,10 @@ public class ClusterInfo {
return this.haState.toString();
}
+ public String getRMStateStore() {
+ return this.rmStateStoreName;
+ }
+
public String getRMVersion() {
return this.resourceManagerVersion;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25df42d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 561b147..ff0f6f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -284,7 +284,7 @@ public class TestRMWebServices extends JerseyTest {
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("clusterInfo");
- assertEquals("incorrect number of elements", 10, info.length());
+ assertEquals("incorrect number of elements", 11, info.length());
verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"),
info.getString("state"), info.getString("haState"),
info.getString("hadoopVersionBuiltOn"),